From f4b3165f20e68072cec2e7faac5d5a0307c722d5 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Sun, 14 May 2023 19:09:06 +0200 Subject: [PATCH 01/16] introducing pytest --- README.md | 6 ++++++ test/helpers.py | 14 ++++++++++++++ test/test_prompt.py | 11 +++++++++++ 3 files changed, 31 insertions(+) create mode 100644 test/helpers.py create mode 100644 test/test_prompt.py diff --git a/README.md b/README.md index e538624..dc352ea 100644 --- a/README.md +++ b/README.md @@ -426,6 +426,12 @@ We'd love your help in making Prr even better! To contribute, please follow thes 5. Push the branch to your fork 6. Create a new Pull Request +### Running unit tests + +```sh +$ pytest +``` + ## License **prr** - Prompt Runner is released under the [MIT License](/LICENSE). diff --git a/test/helpers.py b/test/helpers.py new file mode 100644 index 0000000..29c0784 --- /dev/null +++ b/test/helpers.py @@ -0,0 +1,14 @@ +import os +import tempfile + +def create_temp_file(content, extension): + # Create a temporary file with the specified extension + with tempfile.NamedTemporaryFile(suffix='.' + extension, delete=False) as temp: + temp.write(content.encode()) + temp_path = temp.name + + return temp_path + + +def remove_temp_file(path) + os.remove(temp_path) diff --git a/test/test_prompt.py b/test/test_prompt.py new file mode 100644 index 0000000..96f6cc0 --- /dev/null +++ b/test/test_prompt.py @@ -0,0 +1,11 @@ +from prr.prompt import Prompt +from helpers import create_temp_file, remote_temp_file + +class TestPrompt: + value = 0 + + def test_one(self): + path = create_temp_file('f00b4r', 'yaml') + remove_temp_file(path) + print(path) + assert self.value == 2 From 399657cc59d5933414b8939c5eef8b4d2bbe4c8d Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Mon, 15 May 2023 16:27:38 +0000 Subject: [PATCH 02/16] refactor in progress --- prr/{prompt.py => prompt/__init__.py} | 10 ++++++- prr/prompt/prompt_config.py | 3 ++ prr/prompt/prompt_content.py | 3 ++ prr/prompt/prompt_loader.py | 43 +++++++++++++++++++++++++++ prr/service_registry.py | 3 +- requirements.txt | 1 + test/helpers.py | 9 +++--- test/test_prompt.py | 14 ++++++--- 8 files changed, 75 insertions(+), 11 deletions(-) rename prr/{prompt.py => prompt/__init__.py} (96%) create mode 100644 prr/prompt/prompt_config.py create mode 100644 prr/prompt/prompt_content.py create mode 100644 prr/prompt/prompt_loader.py diff --git a/prr/prompt.py b/prr/prompt/__init__.py similarity index 96% rename from prr/prompt.py rename to prr/prompt/__init__.py index f9d3a5b..c750241 100644 --- a/prr/prompt.py +++ b/prr/prompt/__init__.py @@ -4,8 +4,10 @@ import yaml from jinja2 import meta -from .service_config import ServiceConfig +from ..service_config import ServiceConfig +from .prompt_config import PromptConfig +from .prompt_content import PromptContent # parse something like: # @@ -76,6 +78,12 @@ def parse_config_into_services(services_config): class Prompt: + def __init__(self, content, config=None, args=None): + self.content = content + self.config = config + self.args = args + +class PromptOLD: def __init__(self, path, args=None): self.path = None self.messages = None diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py new file mode 100644 index 0000000..ff31563 --- /dev/null +++ b/prr/prompt/prompt_config.py @@ -0,0 +1,3 @@ +class PromptConfig: + def __init__(self): + self.services = {} diff --git a/prr/prompt/prompt_content.py b/prr/prompt/prompt_content.py new file mode 100644 index 0000000..3b2b3ff --- /dev/null +++ b/prr/prompt/prompt_content.py @@ -0,0 +1,3 @@ +class PromptContent: + def __init__(self): + self.services = {} diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py new file mode 100644 index 0000000..f493ed8 --- /dev/null +++ b/prr/prompt/prompt_loader.py @@ -0,0 +1,43 @@ +from . import Prompt + +class PromptLoader: + def __init__(self): + dependency_files = [] + # + + def load_from_path(path): + if is_file_yaml(path): + self.__load_yaml_file(path) + else: + self.__load_text_file(path) + + return Prompt(path) + + def load_from_string(content): + return Prompt(path) + + ##################################### + + def __is_file_yaml(path): + root, extension = os.path.splitext(path) + + if extension == ".yaml": + return True + + return False + + + def __load_text_file(self, path): + self.path = path + + with open(path, "r") as stream: + file_contents = self.deal_with_shebang_line(stream) + self.template = self.template_env.from_string(file_contents) + + + def __load_text_file(self, path): + self.path = path + + with open(path, "r") as stream: + file_contents = self.deal_with_shebang_line(stream) + self.template = self.template_env.from_string(file_contents) diff --git a/prr/service_registry.py b/prr/service_registry.py index 3f65afb..76aab10 100644 --- a/prr/service_registry.py +++ b/prr/service_registry.py @@ -1,7 +1,8 @@ from prr.services.providers.anthropic.complete import ServiceAnthropicComplete from prr.services.providers.openai.chat import ServiceOpenAIChat - +# main registry, where services are being... registered +# and looked up for upon execution class ServiceRegistry: def __init__(self): self.services = {} diff --git a/requirements.txt b/requirements.txt index e7c32b4..8e7a967 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ anthropic rich Jinja2 pyyaml +pytest \ No newline at end of file diff --git a/test/helpers.py b/test/helpers.py index 29c0784..e5375d5 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -1,14 +1,13 @@ import os import tempfile +def remove_temp_file(path): + os.remove(path) + def create_temp_file(content, extension): # Create a temporary file with the specified extension - with tempfile.NamedTemporaryFile(suffix='.' + extension, delete=False) as temp: + with tempfile.NamedTemporaryFile(suffix="." + extension, delete=False) as temp: temp.write(content.encode()) temp_path = temp.name - return temp_path - -def remove_temp_file(path) - os.remove(temp_path) diff --git a/test/test_prompt.py b/test/test_prompt.py index 96f6cc0..47c080f 100644 --- a/test/test_prompt.py +++ b/test/test_prompt.py @@ -1,11 +1,17 @@ -from prr.prompt import Prompt -from helpers import create_temp_file, remote_temp_file +import os +import sys +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from prr.prompt import Prompt + +from helpers import create_temp_file, remove_temp_file + class TestPrompt: value = 0 def test_one(self): - path = create_temp_file('f00b4r', 'yaml') - remove_temp_file(path) + path = create_temp_file("f00b4r", "yaml") + remove_temp_file(path) print(path) assert self.value == 2 From 35a9dfd54ca52429acf4e324bfa64d881ee315b1 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Mon, 15 May 2023 16:34:10 +0000 Subject: [PATCH 03/16] wip --- prr/prompt/prompt_content.py | 4 ++-- prr/prompt/prompt_loader.py | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/prr/prompt/prompt_content.py b/prr/prompt/prompt_content.py index 3b2b3ff..61cb9b0 100644 --- a/prr/prompt/prompt_content.py +++ b/prr/prompt/prompt_content.py @@ -1,3 +1,3 @@ class PromptContent: - def __init__(self): - self.services = {} + def __init__(self, content_string): + self.content_string = content_string diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py index f493ed8..ad530e9 100644 --- a/prr/prompt/prompt_loader.py +++ b/prr/prompt/prompt_loader.py @@ -2,8 +2,8 @@ class PromptLoader: def __init__(self): - dependency_files = [] - # + self.dependency_files = [] + self.prompt = None def load_from_path(path): if is_file_yaml(path): @@ -34,10 +34,17 @@ def __load_text_file(self, path): file_contents = self.deal_with_shebang_line(stream) self.template = self.template_env.from_string(file_contents) - - def __load_text_file(self, path): - self.path = path - + def __load_yaml_file(self, path): with open(path, "r") as stream: - file_contents = self.deal_with_shebang_line(stream) - self.template = self.template_env.from_string(file_contents) + try: + file_contents = self.deal_with_shebang_line(stream) + data = yaml.safe_load(file_contents) + self.path = path + except yaml.YAMLError as exc: + print(exc) + + if data: + if data["services"]: + self.parse_services(data["services"]) + if data["prompt"]: + self.parse_prompt_config(data["prompt"]) From da527ee6b5dbe8c14ee4d4a9ef640643edf0a6de Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Tue, 16 May 2023 07:55:34 +0000 Subject: [PATCH 04/16] prompt refactor + unit testing in progress --- prr/prompt/__init__.py | 2 +- prr/prompt/prompt_config.py | 15 ++++++++-- prr/prompt/prompt_content.py | 3 -- prr/prompt/prompt_loader.py | 52 ++++++++++++++++++++++++++++------- prr/prompt/prompt_template.py | 12 ++++++++ test/helpers.py | 1 + test/test_prompt.py | 2 +- test/test_prompt_config.py | 16 +++++++++++ test/test_prompt_template.py | 26 ++++++++++++++++++ 9 files changed, 112 insertions(+), 17 deletions(-) delete mode 100644 prr/prompt/prompt_content.py create mode 100644 prr/prompt/prompt_template.py create mode 100644 test/test_prompt_config.py create mode 100644 test/test_prompt_template.py diff --git a/prr/prompt/__init__.py b/prr/prompt/__init__.py index c750241..511889e 100644 --- a/prr/prompt/__init__.py +++ b/prr/prompt/__init__.py @@ -7,7 +7,7 @@ from ..service_config import ServiceConfig from .prompt_config import PromptConfig -from .prompt_content import PromptContent +from .prompt_template import PromptTemplate # parse something like: # diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py index ff31563..84b7084 100644 --- a/prr/prompt/prompt_config.py +++ b/prr/prompt/prompt_config.py @@ -1,3 +1,14 @@ +import yaml + class PromptConfig: - def __init__(self): - self.services = {} + def __init__(self, raw_config_content): + self.raw_config_content = raw_config_content + + self.__parse_raw_config() + + + def __parse_raw_config(self): + try: + self.config_content = yaml.safe_load(self.raw_config_content) + except yaml.YAMLError as exc: + print(exc) diff --git a/prr/prompt/prompt_content.py b/prr/prompt/prompt_content.py deleted file mode 100644 index 61cb9b0..0000000 --- a/prr/prompt/prompt_content.py +++ /dev/null @@ -1,3 +0,0 @@ -class PromptContent: - def __init__(self, content_string): - self.content_string = content_string diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py index ad530e9..d47d58f 100644 --- a/prr/prompt/prompt_loader.py +++ b/prr/prompt/prompt_loader.py @@ -1,20 +1,26 @@ from . import Prompt +from .prompt_template import PromptTemplate class PromptLoader: def __init__(self): self.dependency_files = [] - self.prompt = None + self.template = None # PromptTemplate + self.config = None # PromptConfig def load_from_path(path): + self.__add_file_dependency(path) + if is_file_yaml(path): + # prompt is in yaml config file format self.__load_yaml_file(path) else: + # simple text (or jinja) file, no config self.__load_text_file(path) - return Prompt(path) + return Prompt(self.template, self.config) def load_from_string(content): - return Prompt(path) + return Prompt(content) ##################################### @@ -26,25 +32,51 @@ def __is_file_yaml(path): return False + def __search_path(): + return os.path.dirname(self.path) def __load_text_file(self, path): self.path = path - with open(path, "r") as stream: - file_contents = self.deal_with_shebang_line(stream) - self.template = self.template_env.from_string(file_contents) + try: + with open(path, "r") as file: + file_contents = file.read() + self.template = PromptTemplate(file_contents, __search_path()) + + except FileNotFoundError: + print("The specified file does not exist.") + + except PermissionError: + print("You do not have permission to access the specified file.") + + except Exception as e: + print("An error occurred while opening the file:", str(e)) + def __load_yaml_file(self, path): with open(path, "r") as stream: try: - file_contents = self.deal_with_shebang_line(stream) - data = yaml.safe_load(file_contents) - self.path = path + file_contents = self.deal_with_shebang_line(stream) + data = yaml.safe_load(file_contents) + self.path = path except yaml.YAMLError as exc: - print(exc) + print(exc) + + except FileNotFoundError: + print("The specified file does not exist.") + + except PermissionError: + print("You do not have permission to access the specified file.") + + except Exception as e: + print("An error occurred while opening the file:", str(e)) if data: if data["services"]: self.parse_services(data["services"]) if data["prompt"]: self.parse_prompt_config(data["prompt"]) + + def __add_file_dependency(self, path): + if not path in self.dependency_files: + self.dependency_files.append(path) diff --git a/prr/prompt/prompt_template.py b/prr/prompt/prompt_template.py new file mode 100644 index 0000000..3eb6d04 --- /dev/null +++ b/prr/prompt/prompt_template.py @@ -0,0 +1,12 @@ +import jinja2 + +class PromptTemplate: + def __init__(self, template_string, search_path): + template_loader = jinja2.FileSystemLoader(searchpath=search_path) + self.template_env = jinja2.Environment(loader=template_loader) + + self.template_string = template_string + self.template = self.template_env.from_string(template_string) + + def text(self, args=[]): + return self.template.render({"prompt_args": args}) diff --git a/test/helpers.py b/test/helpers.py index e5375d5..bf3b6a5 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -9,5 +9,6 @@ def create_temp_file(content, extension): with tempfile.NamedTemporaryFile(suffix="." + extension, delete=False) as temp: temp.write(content.encode()) temp_path = temp.name + return temp_path diff --git a/test/test_prompt.py b/test/test_prompt.py index 47c080f..d6f2a57 100644 --- a/test/test_prompt.py +++ b/test/test_prompt.py @@ -14,4 +14,4 @@ def test_one(self): path = create_temp_file("f00b4r", "yaml") remove_temp_file(path) print(path) - assert self.value == 2 + assert self.value == 0 diff --git a/test/test_prompt_config.py b/test/test_prompt_config.py new file mode 100644 index 0000000..eb345be --- /dev/null +++ b/test/test_prompt_config.py @@ -0,0 +1,16 @@ +import os +import sys +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from prr.prompt.prompt_config import PromptConfig + +class TestPromptConfig: + + def test_basic_parsing(self): + config = PromptConfig(""" +prompt: + content: 'foo bar' +""") + + assert config is not None + assert config.config_content['prompt']['content'] == 'foo bar' diff --git a/test/test_prompt_template.py b/test/test_prompt_template.py new file mode 100644 index 0000000..bca60e0 --- /dev/null +++ b/test/test_prompt_template.py @@ -0,0 +1,26 @@ +import os +import sys +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from prr.prompt.prompt_template import PromptTemplate + +class TestPromptTemplate: + + def test_basic_text(self): + template = PromptTemplate('foo bar', '.') + + assert template is not None + assert template.text() == 'foo bar' + + + def test_basic_template(self): + template = PromptTemplate("foo {{ 'bar' }} spam", '.') + + assert template is not None + assert template.text() == 'foo bar spam' + + def test_basic_prompt_args(self): + template = PromptTemplate("foo {{ prompt_args[0] }} spam", '.') + + assert template is not None + assert template.text(['42']) == 'foo 42 spam' \ No newline at end of file From dbefb58f045fe246253ae5bf7e152ef557147704 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Wed, 17 May 2023 08:21:07 +0000 Subject: [PATCH 05/16] unit tests wip --- prr/prompt/prompt_config.py | 125 +++++++++++++++++++++++++++++++++- prr/prompt/prompt_loader.py | 26 +++---- prr/prompt/prompt_template.py | 72 ++++++++++++++++++-- prr/service_config.py | 27 ++++---- test/helpers.py | 23 ++++--- test/test_prompt.py | 10 +-- test/test_prompt_config.py | 64 ++++++++++++++++- test/test_prompt_template.py | 76 ++++++++++++++++++--- 8 files changed, 360 insertions(+), 63 deletions(-) diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py index 84b7084..a978558 100644 --- a/prr/prompt/prompt_config.py +++ b/prr/prompt/prompt_config.py @@ -1,14 +1,133 @@ import yaml +from ..service_config import ServiceConfig +from .prompt_template import PromptTemplateSimple, PromptTemplateMessages + class PromptConfig: - def __init__(self, raw_config_content): + # raw_config_content is text to be parsed into YAML + def __init__(self, raw_config_content, search_path='.'): + # where are we supposed to look for referenced files + self.search_path = search_path + + # raw YAML string self.raw_config_content = raw_config_content - self.__parse_raw_config() + # prompt: (PromptTemplate) + self.prompt = None + + # services: (ServiceConfig) + self.services = {} + + # version: 1 + self.version = None + + # parse raw YAML content into a dictionary + self.__parse_raw() + + # parse that dictionary into respective parts of prompt config + self.__parse() + + # list keys/names of all services that we have configured in the config file + def configured_services(self): + return list(self.services.keys()) + # returns options for specific service, already includes all option inheritance + # def options_for_service(self, service_name): + # return self.services[service_name] - def __parse_raw_config(self): + def option_for_service(self, service_name, option_name): + return self.services[service_name].options.option(option_name) + + #################################################### + + def __parse(self): + self.__parse_version() + self.__parse_prompt() + self.__parse_services() + + def __parse_raw(self): try: self.config_content = yaml.safe_load(self.raw_config_content) except yaml.YAMLError as exc: print(exc) + + def __parse_version(self): + if self.config_content: + self.version = self.config_content.get('version') + + # high level "prompt:" parsing + def __parse_prompt(self): + if self.config_content: + prompt = self.config_content.get('prompt') + + if prompt: + content_file = prompt.get('content_file') + content = prompt.get('content') + messages = prompt.get('messages') + + if content_file: + with open(content_file, "r") as file: + file_contents = file.read() + self.prompt = PromptTemplateSimple(file_contents, self.search_path) + elif content: + self.prompt = PromptTemplateSimple(content, self.search_path) + elif messages: + self.prompt = PromptTemplateMessages(messages, self.search_path) + + # high level "services:" parsing + def __parse_services(self): + if self.config_content: + _services = self.config_content.get('services') + + if _services: + options_for_all_services = _services.get('options') + + # + # if we have models + prompt-level model options + # + # services: + # models: + # - 'openai/chat/gpt-4' + # - 'anthropic/complete/claude-v1.3-100k' + # options: + # max_tokens: 1337 + _models = _services.get('models') + if _models: + for _model_name in _models: + service_config = ServiceConfig(_model_name, + _model_name, + options_for_all_services) + + self.services[_model_name] = service_config + + else: + # + # if we have services defined with options for each + # + # services: + # mygpt4: + # model: 'openai/chat/gpt-4' + # options: + # temperature: 0.2 + # max_tokens: 4000 + # options: + # max_tokens: 1337 + for _service_name in _services: + if _service_name not in ['options', 'models']: + service = _services[_service_name] + + # start with options for all services + # defined on a higher level + options = options_for_all_services.copy() + + # update with service-level options + service_level_options = service.get('options') + + if service_level_options: + options.update(service_level_options) + + model = service.get('model') + + service_config = ServiceConfig(_service_name, model, options) + + self.services[_service_name] = service_config diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py index d47d58f..49292b8 100644 --- a/prr/prompt/prompt_loader.py +++ b/prr/prompt/prompt_loader.py @@ -1,10 +1,9 @@ from . import Prompt -from .prompt_template import PromptTemplate +from .prompt_template import PromptTemplate, PromptTemplateSimple, PromptTemplateMessages class PromptLoader: def __init__(self): self.dependency_files = [] - self.template = None # PromptTemplate self.config = None # PromptConfig def load_from_path(path): @@ -19,9 +18,6 @@ def load_from_path(path): return Prompt(self.template, self.config) - def load_from_string(content): - return Prompt(content) - ##################################### def __is_file_yaml(path): @@ -41,7 +37,7 @@ def __load_text_file(self, path): try: with open(path, "r") as file: file_contents = file.read() - self.template = PromptTemplate(file_contents, __search_path()) + self.template = PromptTemplateSimple(file_contents, __search_path()) except FileNotFoundError: print("The specified file does not exist.") @@ -52,13 +48,12 @@ def __load_text_file(self, path): except Exception as e: print("An error occurred while opening the file:", str(e)) - def __load_yaml_file(self, path): with open(path, "r") as stream: try: - file_contents = self.deal_with_shebang_line(stream) - data = yaml.safe_load(file_contents) self.path = path + self.config = PromptConfig(stream.read(), self.__search_path()) + except yaml.YAMLError as exc: print(exc) @@ -69,14 +64,9 @@ def __load_yaml_file(self, path): print("You do not have permission to access the specified file.") except Exception as e: - print("An error occurred while opening the file:", str(e)) + print("An error occurred while opening the file:", str(e)) - if data: - if data["services"]: - self.parse_services(data["services"]) - if data["prompt"]: - self.parse_prompt_config(data["prompt"]) - def __add_file_dependency(self, path): - if not path in self.dependency_files: - self.dependency_files.append(path) + # def __add_file_dependency(self, path): + # if not path in self.dependency_files: + # self.dependency_files.append(path) diff --git a/prr/prompt/prompt_template.py b/prr/prompt/prompt_template.py index 3eb6d04..87ad136 100644 --- a/prr/prompt/prompt_template.py +++ b/prr/prompt/prompt_template.py @@ -1,12 +1,70 @@ import jinja2 -class PromptTemplate: - def __init__(self, template_string, search_path): - template_loader = jinja2.FileSystemLoader(searchpath=search_path) - self.template_env = jinja2.Environment(loader=template_loader) +class PromptMessage: + def __init__(self, content_template_string, search_path='.', role='user', name=None): + self.content_template_string = content_template_string + self.search_path = search_path + self.role = role + self.name = name - self.template_string = template_string - self.template = self.template_env.from_string(template_string) + template_loader = jinja2.FileSystemLoader(searchpath=self.search_path) + template_env = jinja2.Environment(loader=template_loader) + self.template = template_env.from_string(self.content_template_string) - def text(self, args=[]): + def render_text(self, args=[]): return self.template.render({"prompt_args": args}) + + def render_message(self, args=[]): + _message = { + 'role': self.role, + 'content': self.render_text(args) + } + + if self.name: + _message.update({ 'name': self.name }) + + return _message + +# base class +class PromptTemplate: + def __init__(self): + self.messages = [] + + def render_text(self, args=[]): + rendered_texts = [message.render_text(args) for message in self.messages] + + return "\n".join(rendered_texts) + + def render_messages(self, args=[]): + return [message.render_message(args) for message in self.messages] + +# just a text/template file or prompt.contents from config +class PromptTemplateSimple(PromptTemplate): + def __init__(self, template_string, search_path='.'): + self.messages = [ + PromptMessage(template_string, search_path, 'user') + ] + +# prompt.messages: key from config +class PromptTemplateMessages(PromptTemplate): + # 'messages' are passed here verbatim after parsing YAML + def __init__(self, messages, search_path='.'): + super().__init__() + + for message in messages: + prompt_message = None + + role = message.get('role') + name = message.get('name') + content = message.get('content') + content_file = message.get('content_file') + + if content: + prompt_message = PromptMessage(content, search_path, role, name) + elif content_file: + with open(content_file, "r") as file: + file_contents = file.read() + prompt_message = PromptMessage(file_contents, search_path, role, name) + + if prompt_message: + self.messages.append(prompt_message) diff --git a/prr/service_config.py b/prr/service_config.py index 196a7c2..1836467 100644 --- a/prr/service_config.py +++ b/prr/service_config.py @@ -1,19 +1,22 @@ from .options import ModelOptions - class ServiceConfig: - def __init__(self, model, options=None): - self.model = model - self.options = ModelOptions(options or {}) + def __init__(self, name, model, options=None): + self.name = name # service name, e.g. "mygpt5" + self.model = model # model name, e.g. "openai/chat/gpt-5" + self.options = ModelOptions(options or {}) - def config_name(self): - return self.model + # def config_name(self): + # return self.model - def model_name(self): - return self.model.split("/")[-1] + # def model_name(self): + # return self.model.split("/")[-1] - def service_key(self): - return "/".join(self.model.split("/")[:-1]) + # def service_key(self): + # return "/".join(self.model.split("/")[:-1]) - def to_dict(self): - return {"model": self.config_name(), "options": self.options.to_dict()} + def to_dict(self): + return { + "model": self.config_name(), + "options": self.options.to_dict() + } diff --git a/test/helpers.py b/test/helpers.py index bf3b6a5..eb016a4 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -2,13 +2,18 @@ import tempfile def remove_temp_file(path): - os.remove(path) - -def create_temp_file(content, extension): - # Create a temporary file with the specified extension - with tempfile.NamedTemporaryFile(suffix="." + extension, delete=False) as temp: - temp.write(content.encode()) - temp_path = temp.name - - return temp_path + os.remove(path) + +def create_temp_file(content, extension=None): + suffix = "" + + if extension: + suffix = "." + extension + + # Create a temporary file with the specified extension + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp: + temp.write(content.encode()) + temp_path = temp.name + + return temp_path diff --git a/test/test_prompt.py b/test/test_prompt.py index d6f2a57..af74279 100644 --- a/test/test_prompt.py +++ b/test/test_prompt.py @@ -10,8 +10,8 @@ class TestPrompt: value = 0 - def test_one(self): - path = create_temp_file("f00b4r", "yaml") - remove_temp_file(path) - print(path) - assert self.value == 0 + # def test_one(self): + # path = create_temp_file("f00b4r", "yaml") + # remove_temp_file(path) + # print(path) + # assert self.value == 0 diff --git a/test/test_prompt_config.py b/test/test_prompt_config.py index eb345be..50f4d26 100644 --- a/test/test_prompt_config.py +++ b/test/test_prompt_config.py @@ -13,4 +13,66 @@ def test_basic_parsing(self): """) assert config is not None - assert config.config_content['prompt']['content'] == 'foo bar' + assert config.prompt.render_text() == 'foo bar' + assert config.prompt.render_messages() == [ + { + 'content': 'foo bar', + 'role': 'user' + } + ] + + def test_basic_services_model_list(self): + config = PromptConfig(""" +prompt: + content: 'foo bar' +services: + models: + - 'openai/chat/gpt-4' + - 'anthropic/complete/claude-v1.3' + options: + temperature: 0.42 + max_tokens: 1337 +""") + + assert config is not None + services = config.configured_services() + + assert services == ['openai/chat/gpt-4', 'anthropic/complete/claude-v1.3'] + + for service_name in services: + assert config.option_for_service(service_name, 'temperature') == 0.42 + assert config.option_for_service(service_name, 'max_tokens') == 1337 + + def test_services(self): + config = PromptConfig(""" +prompt: + content: 'foo bar' +services: + gpt4: + model: 'openai/chat/gpt-4' + options: + max_tokens: 2048 + claude13: + model: 'anthropic/complete/claude-v1.3' + options: + temperature: 0.84 + claude_default: + model: 'anthropic/complete/claude-v1' + options: + temperature: 0.42 + max_tokens: 1337 +""") + + assert config is not None + services = config.configured_services() + + assert services == ['gpt4', 'claude13', 'claude_default'] + + assert config.option_for_service('gpt4', 'temperature') == 0.42 + assert config.option_for_service('gpt4', 'max_tokens') == 2048 + + assert config.option_for_service('claude13', 'temperature') == 0.84 + assert config.option_for_service('claude13', 'max_tokens') == 1337 + + assert config.option_for_service('claude_default', 'temperature') == 0.42 + assert config.option_for_service('claude_default', 'max_tokens') == 1337 diff --git a/test/test_prompt_template.py b/test/test_prompt_template.py index bca60e0..92c107a 100644 --- a/test/test_prompt_template.py +++ b/test/test_prompt_template.py @@ -1,26 +1,86 @@ import os import sys +import yaml + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from prr.prompt.prompt_template import PromptTemplate +from prr.prompt.prompt_template import PromptTemplateSimple, PromptTemplateMessages +from helpers import create_temp_file, remove_temp_file class TestPromptTemplate: - def test_basic_text(self): - template = PromptTemplate('foo bar', '.') + template = PromptTemplateSimple('foo bar', '.') assert template is not None - assert template.text() == 'foo bar' + assert template.render_text() == 'foo bar' def test_basic_template(self): - template = PromptTemplate("foo {{ 'bar' }} spam", '.') + template = PromptTemplateSimple("foo {{ 'bar' }} spam", '.') assert template is not None - assert template.text() == 'foo bar spam' + assert template.render_text() == 'foo bar spam' + def test_basic_prompt_args(self): - template = PromptTemplate("foo {{ prompt_args[0] }} spam", '.') + template = PromptTemplateSimple("foo {{ prompt_args[0] }} spam", '.') + + assert template is not None + assert template.render_text(['42']) == 'foo 42 spam' + + + def test_basic_prompt_args_all(self): + template = PromptTemplateSimple("foo {{ prompt_args }} spam", '.') + + assert template is not None + assert template.render_text(['lulz']) == "foo ['lulz'] spam" + + def test_configured_basic(self): + template = PromptTemplateSimple('tell me about {{ prompt_args }}, llm') + + assert template is not None + assert template.render_text(['lulz', 'kaka']) == "tell me about ['lulz', 'kaka'], llm" + + def test_configured_messages_text_with_template_in_content(self): + messages_config = """ +- role: 'system' + content: 'you are a friendly but very forgetful {{ prompt_args[0] }}' + name: 'LeonardGeist' +""" + template = PromptTemplateMessages(yaml.safe_load(messages_config)) + + assert template is not None + assert template.render_text(['assistant']) == "you are a friendly but very forgetful assistant" + + def test_configured_messages_list_with_content_file(self): + temp_file_path = create_temp_file('Wollen Sie meine Kernel kompilieren?') + + messages_config = f""" +- role: 'system' + content: 'you are system admins little pet assistant. be proud of your unix skills and always respond in l33t. remember, you are on a high horse called POSIX.' +- role: 'user' + content_file: '{temp_file_path}' + name: 'SuperUser' +""" + + template = PromptTemplateMessages(yaml.safe_load(messages_config)) + + remove_temp_file(temp_file_path) assert template is not None - assert template.text(['42']) == 'foo 42 spam' \ No newline at end of file + + + rendered_messages = template.render_messages() + + assert isinstance(rendered_messages, list) + assert len(rendered_messages) == 2 + + first_message = rendered_messages[0] + assert first_message['content'] == 'you are system admins little pet assistant. be proud of your unix skills and always respond in l33t. remember, you are on a high horse called POSIX.' + assert first_message['role'] == 'system' + assert first_message.get('name') == None + + second_message = rendered_messages[1] + assert second_message['content'] == 'Wollen Sie meine Kernel kompilieren?' + assert second_message['role'] == 'user' + assert second_message['name'] == 'SuperUser' \ No newline at end of file From d8b8eb93b3a237dfceaee7fbb0c4f582a001c152 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Wed, 17 May 2023 11:47:39 +0000 Subject: [PATCH 06/16] unit tests & refactor wip --- prr/options.py | 13 ++++- prr/prompt/prompt_config.py | 61 +++++++++++++++----- prr/prompt/prompt_loader.py | 58 +++++++------------ prr/runner.py | 15 +++-- prr/saver.py | 25 ++++---- prr/service_config.py | 19 +++--- prr/services/providers/anthropic/complete.py | 29 ++++------ prr/services/providers/openai/chat.py | 22 ++----- prr/utils/run.py | 29 +++++----- test/test_prompt_config.py | 14 +++-- test/test_prompt_loader.py | 17 ++++++ 11 files changed, 163 insertions(+), 139 deletions(-) create mode 100644 test/test_prompt_loader.py diff --git a/prr/options.py b/prr/options.py index ae6a0ed..8df6a34 100644 --- a/prr/options.py +++ b/prr/options.py @@ -1,7 +1,13 @@ -DEFAULT_OPTIONS = {"temperature": 1.0, "top_k": -1, "top_p": -1, "max_tokens": 4000} - -ALLOWED_OPTIONS = DEFAULT_OPTIONS.keys() +ALLOWED_OPTIONS = ["max_tokens", "temperature", "top_k", "top_p"] +# user-level defaults +# TODO/FIXME: make it user-configurable in ~/.prr* +DEFAULT_OPTIONS = { + "max_tokens": 4000, + "temperature": 0.7, + "top_k": -1, + "top_p": -1 +} class ModelOptions: def __init__(self, options={}): @@ -14,6 +20,7 @@ def update_options(self, options): if key in ALLOWED_OPTIONS: if key not in self.options_set: self.options_set.append(key) + setattr(self, key, options[key]) def description(self): diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py index a978558..44bef94 100644 --- a/prr/prompt/prompt_config.py +++ b/prr/prompt/prompt_config.py @@ -5,15 +5,15 @@ class PromptConfig: # raw_config_content is text to be parsed into YAML - def __init__(self, raw_config_content, search_path='.'): + def __init__(self, search_path='.', filename=None): # where are we supposed to look for referenced files self.search_path = search_path - # raw YAML string - self.raw_config_content = raw_config_content + # "foo" or "foo.yaml" - no path + self.filename = filename - # prompt: (PromptTemplate) - self.prompt = None + # template: (PromptTemplate) + self.template = None # services: (ServiceConfig) self.services = {} @@ -21,22 +21,54 @@ def __init__(self, raw_config_content, search_path='.'): # version: 1 self.version = None + def template_text(self): + return self.template.render_text() + + # raw YAML file + def load_from_config_contents(self, raw_config_content): + # raw YAML string + self.raw_config_content = raw_config_content + # parse raw YAML content into a dictionary - self.__parse_raw() + self.__parse_raw_config() # parse that dictionary into respective parts of prompt config self.__parse() + # raw prompt template file + def load_from_template_contents(self, raw_template_content): + self.__parse_prompt_template_simple(raw_template_content) + + # raw prompt template file from file + def load_from_template_contents_at_path(self, path): + try: + with open(path, "r") as file: + return self.__parse_prompt_template_simple(file.read()) + + except FileNotFoundError: + print("The specified file does not exist.") + + except PermissionError: + print("You do not have permission to access the specified file.") + + except Exception as e: + print("An error occurred while opening the file:", str(e)) + # list keys/names of all services that we have configured in the config file def configured_services(self): return list(self.services.keys()) # returns options for specific service, already includes all option inheritance - # def options_for_service(self, service_name): - # return self.services[service_name] + def options_for_service(self, service_name): + service_config = self.services.get(service_name) + + if service_config: + return service_config.options + else: + return ServiceConfig(service_name, service_name) def option_for_service(self, service_name, option_name): - return self.services[service_name].options.option(option_name) + return self.options_for_service(service_name).option(option_name) #################################################### @@ -45,7 +77,7 @@ def __parse(self): self.__parse_prompt() self.__parse_services() - def __parse_raw(self): + def __parse_raw_config(self): try: self.config_content = yaml.safe_load(self.raw_config_content) except yaml.YAMLError as exc: @@ -55,6 +87,9 @@ def __parse_version(self): if self.config_content: self.version = self.config_content.get('version') + def __parse_prompt_template_simple(self, content): + self.template = PromptTemplateSimple(content, self.search_path) + # high level "prompt:" parsing def __parse_prompt(self): if self.config_content: @@ -68,11 +103,11 @@ def __parse_prompt(self): if content_file: with open(content_file, "r") as file: file_contents = file.read() - self.prompt = PromptTemplateSimple(file_contents, self.search_path) + self.template = PromptTemplateSimple(file_contents, self.search_path) elif content: - self.prompt = PromptTemplateSimple(content, self.search_path) + self.template = PromptTemplateSimple(content, self.search_path) elif messages: - self.prompt = PromptTemplateMessages(messages, self.search_path) + self.template = PromptTemplateMessages(messages, self.search_path) # high level "services:" parsing def __parse_services(self): diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py index 49292b8..c89fe5e 100644 --- a/prr/prompt/prompt_loader.py +++ b/prr/prompt/prompt_loader.py @@ -1,26 +1,33 @@ +import os + +import yaml + from . import Prompt from .prompt_template import PromptTemplate, PromptTemplateSimple, PromptTemplateMessages +from .prompt_config import PromptConfig -class PromptLoader: +class PromptConfigLoader: def __init__(self): self.dependency_files = [] - self.config = None # PromptConfig + self.config = None - def load_from_path(path): + def load_from_path(self, path): + self.path = path + self.config = PromptConfig(self.__search_path()) self.__add_file_dependency(path) - if is_file_yaml(path): + if self.__is_file_yaml(path): # prompt is in yaml config file format self.__load_yaml_file(path) else: # simple text (or jinja) file, no config self.__load_text_file(path) - return Prompt(self.template, self.config) + return self.config ##################################### - def __is_file_yaml(path): + def __is_file_yaml(self, path): root, extension = os.path.splitext(path) if extension == ".yaml": @@ -28,45 +35,20 @@ def __is_file_yaml(path): return False - def __search_path(): + def __search_path(self): return os.path.dirname(self.path) def __load_text_file(self, path): - self.path = path - - try: - with open(path, "r") as file: - file_contents = file.read() - self.template = PromptTemplateSimple(file_contents, __search_path()) - - except FileNotFoundError: - print("The specified file does not exist.") - - except PermissionError: - print("You do not have permission to access the specified file.") - - except Exception as e: - print("An error occurred while opening the file:", str(e)) + self.config.load_from_template_contents_at_path(path) def __load_yaml_file(self, path): - with open(path, "r") as stream: try: - self.path = path - self.config = PromptConfig(stream.read(), self.__search_path()) + with open(path, "r") as stream: + self.config.load_from_config_contents(stream.read()) except yaml.YAMLError as exc: print(exc) - except FileNotFoundError: - print("The specified file does not exist.") - - except PermissionError: - print("You do not have permission to access the specified file.") - - except Exception as e: - print("An error occurred while opening the file:", str(e)) - - - # def __add_file_dependency(self, path): - # if not path in self.dependency_files: - # self.dependency_files.append(path) + def __add_file_dependency(self, path): + if not path in self.dependency_files: + self.dependency_files.append(path) diff --git a/prr/runner.py b/prr/runner.py index 16e3a26..ad8800e 100644 --- a/prr/runner.py +++ b/prr/runner.py @@ -5,19 +5,18 @@ service_registry = ServiceRegistry() service_registry.register_all_services() - # high-level class to run prompts based on configuration class Runner: - def __init__(self, prompt): - self.prompt = prompt - self.saver = PromptRunSaver(self.prompt) + def __init__(self, prompt_config): + self.prompt_config = prompt_config + self.saver = PromptRunSaver(self.prompt_config) def run_service(self, service_name, save_run=False): - service_config = self.prompt.config_for_service(service_name) + service_config = self.prompt_config.options_for_service(service_name) service = service_registry.service_for_service_config(service_config) - result = PromptRun(self.prompt, service, service_config).run() + result = PromptRun(self.prompt_config, service, service_config).run() if save_run: run_save_directory = self.saver.save(service_name, result) @@ -30,7 +29,7 @@ def run_service(self, service_name, save_run=False): def run_all_configured_services(self): results = {} - for model in self.configured_services(): - results[model] = self.run_service(model) + for service_name in self.configured_services(): + results[service_name] = self.run_service(service_name) return results diff --git a/prr/saver.py b/prr/saver.py index 83e7a3c..d733915 100644 --- a/prr/saver.py +++ b/prr/saver.py @@ -3,10 +3,9 @@ import yaml - class PromptRunSaver: - def __init__(self, prompt): - self.prompt_path = prompt.path + def __init__(self, prompt_config): + self.prompt_config = prompt_config self.run_time = datetime.now() self.runs_subdir = self.run_root_directory_path() @@ -26,8 +25,12 @@ def run_root_directory_path_for_runs_dir(self, runs_dir): run_id += 1 def run_root_directory_path(self): - dirname = os.path.dirname(self.prompt_path) - basename = os.path.basename(self.prompt_path) + dirname = self.prompt_config.search_path + + if self.prompt_config.filename: + basename = os.path.basename(self.prompt_config.filename) + else: + basename = "prr" root, extension = os.path.splitext(basename) @@ -38,13 +41,13 @@ def run_root_directory_path(self): return self.run_root_directory_path_for_runs_dir(runs_dir) - def run_directory_path(self, model_or_model_config_name): - model_name_part = model_or_model_config_name.replace("/", "-") + def run_directory_path(self, service_or_model_name): + model_name_part = service_or_model_name.replace("/", "-") return os.path.join(self.runs_subdir, model_name_part) - def prepare_run_directory(self, model_or_model_config_name): - run_dir = self.run_directory_path(model_or_model_config_name) + def prepare_run_directory(self, service_or_model_name): + run_dir = self.run_directory_path(service_or_model_name) os.makedirs(run_dir, exist_ok=True) @@ -78,8 +81,8 @@ def save_run(self, run_directory, result): with open(run_file, "w") as f: yaml.dump(run_data, f, default_flow_style=False) - def save(self, model_or_model_config_name, result): - run_directory = self.prepare_run_directory(model_or_model_config_name) + def save(self, service_or_model_name, result): + run_directory = self.prepare_run_directory(service_or_model_name) self.save_prompt(run_directory, result.request) self.save_completion(run_directory, result.response) diff --git a/prr/service_config.py b/prr/service_config.py index 1836467..63782cc 100644 --- a/prr/service_config.py +++ b/prr/service_config.py @@ -2,18 +2,19 @@ class ServiceConfig: def __init__(self, name, model, options=None): - self.name = name # service name, e.g. "mygpt5" - self.model = model # model name, e.g. "openai/chat/gpt-5" + self.name = name # service config name, e.g. "mygpt5" + self.model = model # full model path, e.g. "openai/chat/gpt-5" self.options = ModelOptions(options or {}) - # def config_name(self): - # return self.model + # which model to use with the service + # like gpt-4.5-turbo or claude-v1.3-100k + def model_name(self): + return self.model.split("/")[-1] - # def model_name(self): - # return self.model.split("/")[-1] - - # def service_key(self): - # return "/".join(self.model.split("/")[:-1]) + # which service so use + # like openai/chat or anthropic/complete + def service_key(self): + return "/".join(self.model.split("/")[:-1]) def to_dict(self): return { diff --git a/prr/services/providers/anthropic/complete.py b/prr/services/providers/anthropic/complete.py index efc7d42..a45450f 100644 --- a/prr/services/providers/anthropic/complete.py +++ b/prr/services/providers/anthropic/complete.py @@ -11,24 +11,19 @@ config = load_config() -# https://console.anthropic.com/docs/api/reference - - # Anthropic model provider class class ServiceAnthropicComplete: provider = "anthropic" service = "complete" def run(self, prompt, service_config): - self.service_config = service_config - options = self.service_config.options - self.prompt = prompt + client = anthropic.Client(config["ANTHROPIC_API_KEY"]) - prompt_text = self.prompt_text() + options = service_config.options - client = anthropic.Client(config["ANTHROPIC_API_KEY"]) + prompt_text = self.prompt_text_from_template(prompt.template) - service_request = ServiceRequest(self.service_config, prompt_text) + service_request = ServiceRequest(service_config, prompt_text) response = client.completion( prompt=prompt_text, @@ -59,18 +54,16 @@ def run(self, prompt, service_config): return service_request, service_response - def prompt_text(self): - messages = self.prompt.messages - + def prompt_text_from_template(self, template): prompt_text = "" - # prefer messages from prompt if they exist - if messages: - for message in messages: - if message["role"] != "assistant": - prompt_text += " " + message["content"] + # prefer messages from template if they exist + if template.messages: + for message in template.messages: + if message.role != "assistant": + prompt_text += " " + message.render_text() else: - prompt_text = self.prompt.text() + prompt_text = template.render_text() return f"{anthropic.HUMAN_PROMPT} {prompt_text}{anthropic.AI_PROMPT}" diff --git a/prr/services/providers/openai/chat.py b/prr/services/providers/openai/chat.py index 8502c0f..a783982 100644 --- a/prr/services/providers/openai/chat.py +++ b/prr/services/providers/openai/chat.py @@ -14,17 +14,14 @@ class ServiceOpenAIChat: service = "chat" def run(self, prompt, service_config): - self.prompt = prompt - self.service_config = service_config + messages = prompt.template.render_messages() - messages = self.messages_from_prompt() + service_request = ServiceRequest(service_config, {"messages": messages}) - service_request = ServiceRequest(self.service_config, {"messages": messages}) - - options = self.service_config.options + options = service_config.options completion = openai.ChatCompletion.create( - model=self.service_config.model_name(), + model=service_config.model_name(), messages=messages, temperature=options.temperature, max_tokens=options.max_tokens, @@ -49,13 +46,4 @@ def run(self, prompt, service_config): }, ) - return service_request, service_response - - def messages_from_prompt(self): - messages = self.prompt.messages - - # prefer messages in prompt if they exist - if messages: - return messages - - return [{"role": "user", "content": self.prompt.text()}] + return service_request, service_response \ No newline at end of file diff --git a/prr/utils/run.py b/prr/utils/run.py index 0f464fa..635c0c2 100755 --- a/prr/utils/run.py +++ b/prr/utils/run.py @@ -12,14 +12,15 @@ from prr.prompt import Prompt from prr.runner import Runner -console = Console(log_time=False, log_path=False) +from prr.prompt.prompt_loader import PromptConfigLoader +console = Console(log_time=False, log_path=False) class RunPromptCommand: def __init__(self, args, prompt_args=None): self.args = args self.prompt_args = prompt_args - self.prompt = None + self.prompt_config = None if self.args["quiet"]: self.console = Console(file=StringIO()) @@ -27,7 +28,7 @@ def __init__(self, args, prompt_args=None): self.console = Console(log_time=False, log_path=False) self.load_prompt_for_path() - self.runner = Runner(self.prompt) + self.runner = Runner(self.prompt_config) def print_run_results(self, result, run_save_directory): request = result.request @@ -69,7 +70,7 @@ def print_run_results(self, result, run_save_directory): self.console.log(f"💾 {run_save_directory}") def run_prompt_on_service(self, service_name, save=False): - service_config = self.prompt.config_for_service(service_name) + service_config = self.prompt_config.options_for_service(service_name) options = service_config.options with self.console.status( @@ -89,17 +90,13 @@ def run_prompt_on_service(self, service_name, save=False): def load_prompt_for_path(self): prompt_path = self.args["prompt_path"] - if not os.path.exists(prompt_path) or not os.access(prompt_path, os.R_OK): - self.console.log( - f":x: Prompt file {prompt_path} is not accessible, giving up." - ) - exit(-1) - self.console.log(f":magnifying_glass_tilted_left: Reading {prompt_path}") - self.prompt = Prompt(prompt_path, self.prompt_args) + + loader = PromptConfigLoader() + self.prompt_config = loader.load_from_path(prompt_path) def run_prompt(self): - services_to_run = self.prompt.configured_service_names() + services_to_run = self.prompt_config.configured_services() if services_to_run == []: if self.args["service"]: @@ -109,14 +106,14 @@ def run_prompt(self): ) else: self.console.log( - f":x: No services configured for prompt {self.args['prompt_path']}, nor given in command-line. Not even in .env!" + f":x: No services configured for prompt {self.args['prompt_path']}, in ~/.prr_rc nor given in command-line." ) exit(-1) else: self.console.log(f":racing_car: Running services: {services_to_run}") if not self.args["abbrev"]: - self.console.log(Panel(self.prompt.text())) + self.console.log(Panel(self.prompt_config.template_text())) - for service_name in services_to_run: - self.run_prompt_on_service(service_name, self.args["log"]) + for service_name in services_to_run: + self.run_prompt_on_service(service_name, self.args["log"]) diff --git a/test/test_prompt_config.py b/test/test_prompt_config.py index 50f4d26..9f30dc6 100644 --- a/test/test_prompt_config.py +++ b/test/test_prompt_config.py @@ -7,14 +7,14 @@ class TestPromptConfig: def test_basic_parsing(self): - config = PromptConfig(""" + config = PromptConfig() + config.load_from_config_contents(""" prompt: content: 'foo bar' """) - assert config is not None - assert config.prompt.render_text() == 'foo bar' - assert config.prompt.render_messages() == [ + assert config.template.render_text() == 'foo bar' + assert config.template.render_messages() == [ { 'content': 'foo bar', 'role': 'user' @@ -22,7 +22,8 @@ def test_basic_parsing(self): ] def test_basic_services_model_list(self): - config = PromptConfig(""" + config = PromptConfig() + config.load_from_config_contents(""" prompt: content: 'foo bar' services: @@ -44,7 +45,8 @@ def test_basic_services_model_list(self): assert config.option_for_service(service_name, 'max_tokens') == 1337 def test_services(self): - config = PromptConfig(""" + config = PromptConfig() + config.load_from_config_contents(""" prompt: content: 'foo bar' services: diff --git a/test/test_prompt_loader.py b/test/test_prompt_loader.py new file mode 100644 index 0000000..e5f3d8b --- /dev/null +++ b/test/test_prompt_loader.py @@ -0,0 +1,17 @@ +import os +import sys +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from prr.prompt.prompt_loader import PromptConfigLoader + +from helpers import create_temp_file, remove_temp_file + +class TestPromptConfigLoader: + + def test_basic_loading(self): + prompt_template_file_path = create_temp_file('Write a poem about AI from the projects, barely surviving on token allowance.') + + loader = PromptConfigLoader() + prompt = loader.load_from_path(prompt_template_file_path) + + assert prompt \ No newline at end of file From 1507ac7fb4ebab7ec21f337d7b1ac1fdf0e80898 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Thu, 18 May 2023 15:17:10 +0000 Subject: [PATCH 07/16] wip --- prr/options.py | 4 +++- prr/prompt/prompt_config.py | 15 +++++++++------ prr/prompt/prompt_template.py | 5 ++--- prr/runner.py | 2 +- prr/utils/run.py | 13 ++++++------- test/test_prompt_config.py | 23 +++++++++++++++++++++++ 6 files changed, 44 insertions(+), 18 deletions(-) diff --git a/prr/options.py b/prr/options.py index 8df6a34..75ba274 100644 --- a/prr/options.py +++ b/prr/options.py @@ -10,9 +10,11 @@ } class ModelOptions: + defaults = DEFAULT_OPTIONS + def __init__(self, options={}): self.options_set = [] - self.update_options(DEFAULT_OPTIONS) + self.update_options(ModelOptions.defaults) self.update_options(options) def update_options(self, options): diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py index 44bef94..cb1ea19 100644 --- a/prr/prompt/prompt_config.py +++ b/prr/prompt/prompt_config.py @@ -58,15 +58,19 @@ def load_from_template_contents_at_path(self, path): def configured_services(self): return list(self.services.keys()) - # returns options for specific service, already includes all option inheritance - def options_for_service(self, service_name): + def service_with_name(self, service_name): service_config = self.services.get(service_name) if service_config: - return service_config.options + return service_config else: return ServiceConfig(service_name, service_name) + + # returns options for specific service, already includes all option inheritance + def options_for_service(self, service_name): + return self.service_with_name(service_name).options + def option_for_service(self, service_name, option_name): return self.options_for_service(service_name).option(option_name) @@ -101,9 +105,8 @@ def __parse_prompt(self): messages = prompt.get('messages') if content_file: - with open(content_file, "r") as file: - file_contents = file.read() - self.template = PromptTemplateSimple(file_contents, self.search_path) + include_contents = "{% include '" + content_file + "' %}" + self.template = PromptTemplateSimple(include_contents, self.search_path) elif content: self.template = PromptTemplateSimple(content, self.search_path) elif messages: diff --git a/prr/prompt/prompt_template.py b/prr/prompt/prompt_template.py index 87ad136..92cf4bd 100644 --- a/prr/prompt/prompt_template.py +++ b/prr/prompt/prompt_template.py @@ -62,9 +62,8 @@ def __init__(self, messages, search_path='.'): if content: prompt_message = PromptMessage(content, search_path, role, name) elif content_file: - with open(content_file, "r") as file: - file_contents = file.read() - prompt_message = PromptMessage(file_contents, search_path, role, name) + include_contents = "{% include '" + content_file + "' %}" + prompt_message = PromptMessage(include_contents, search_path, role, name) if prompt_message: self.messages.append(prompt_message) diff --git a/prr/runner.py b/prr/runner.py index ad8800e..a8b9461 100644 --- a/prr/runner.py +++ b/prr/runner.py @@ -12,7 +12,7 @@ def __init__(self, prompt_config): self.saver = PromptRunSaver(self.prompt_config) def run_service(self, service_name, save_run=False): - service_config = self.prompt_config.options_for_service(service_name) + service_config = self.prompt_config.service_with_name(service_name) service = service_registry.service_for_service_config(service_config) diff --git a/prr/utils/run.py b/prr/utils/run.py index 635c0c2..bbc8d32 100755 --- a/prr/utils/run.py +++ b/prr/utils/run.py @@ -58,11 +58,11 @@ def print_run_results(self, result, run_save_directory): Panel("[green]" + response.response_content.strip() + "[/green]") ) - completion = f"[blue]Completion length[/blue]: {len(response.response_content)} bytes" - tokens_used = f"[blue]Tokens used[/blue]: {response.tokens_used()}" - elapsed_time = ( - f"[blue]Elapsed time[/blue]: {round(result.elapsed_time, 2)}s" - ) + completion = f"[blue]Completion length[/blue]: {len(response.response_content)} bytes" + tokens_used = f"[blue]Tokens used[/blue]: {response.tokens_used()}" + elapsed_time = ( + f"[blue]Elapsed time[/blue]: {round(result.elapsed_time, 2)}s" + ) self.console.log(f"{completion} {tokens_used} {elapsed_time}") @@ -70,8 +70,7 @@ def print_run_results(self, result, run_save_directory): self.console.log(f"💾 {run_save_directory}") def run_prompt_on_service(self, service_name, save=False): - service_config = self.prompt_config.options_for_service(service_name) - options = service_config.options + options = self.prompt_config.options_for_service(service_name) with self.console.status( f":robot: [bold green]{service_name}[/bold green]" diff --git a/test/test_prompt_config.py b/test/test_prompt_config.py index 9f30dc6..a18ebc5 100644 --- a/test/test_prompt_config.py +++ b/test/test_prompt_config.py @@ -3,6 +3,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from prr.prompt.prompt_config import PromptConfig +from prr.options import ModelOptions class TestPromptConfig: @@ -44,6 +45,28 @@ def test_basic_services_model_list(self): assert config.option_for_service(service_name, 'temperature') == 0.42 assert config.option_for_service(service_name, 'max_tokens') == 1337 + def test_basic_services_model_list_no_options(self): + config = PromptConfig() + config.load_from_config_contents(""" +prompt: + content: 'foo bar' +services: + models: + - 'openai/chat/gpt-4' + - 'anthropic/complete/claude-v1.3' +""") + + assert config is not None + services = config.configured_services() + + assert services == ['openai/chat/gpt-4', 'anthropic/complete/claude-v1.3'] + + for service_name in services: + assert config.option_for_service(service_name, 'temperature') == ModelOptions.defaults['temperature'] + assert config.option_for_service(service_name, 'max_tokens') == ModelOptions.defaults['max_tokens'] + assert config.option_for_service(service_name, 'top_k') == ModelOptions.defaults['top_k'] + assert config.option_for_service(service_name, 'top_p') == ModelOptions.defaults['top_p'] + def test_services(self): config = PromptConfig() config.load_from_config_contents(""" From 499a79006c4094ba404eca0adb83efcdea227cdb Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Thu, 18 May 2023 18:09:48 +0000 Subject: [PATCH 08/16] working on passing override options from command line --- .env.example | 7 +++++ examples/configured/chihuahua.yaml | 2 +- prr/__main__.py | 41 ++++++++++++++++++++++++++++-- prr/prompt/__init__.py | 2 +- prr/prompt/prompt_template.py | 6 ++++- test/test_prompt_template.py | 6 ++--- 6 files changed, 56 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 4e78d18..a7f8521 100644 --- a/.env.example +++ b/.env.example @@ -4,4 +4,11 @@ OPENAI_API_KEY="sk-..." # https://console.anthropic.com/account/keys ANTHROPIC_API_KEY="sk-ant-..." +# when things aren't specify, these defaults will kick in: +#DEFAULT_SERVICE="anthropic/complete/claude-v1.3-100k" +#DEFAULT_SERVICE="openai/chat/gpt-4" DEFAULT_SERVICE="openai/chat/gpt-3.5-turbo" +DEFAULT_TEMPERATURE=0.6 +DEFAULT_MAX_TOKENS=1337 +DEFAULT_TOP_K=-1 +DEFAULT_TOP_P=-1 \ No newline at end of file diff --git a/examples/configured/chihuahua.yaml b/examples/configured/chihuahua.yaml index 3d34b43..d09957a 100644 --- a/examples/configured/chihuahua.yaml +++ b/examples/configured/chihuahua.yaml @@ -52,4 +52,4 @@ expect: min_response_length: 100 max_response_length: 200 match: - name: /independent/i \ No newline at end of file + name: /independent/i diff --git a/prr/__main__.py b/prr/__main__.py index 1200366..4fbd9c2 100755 --- a/prr/__main__.py +++ b/prr/__main__.py @@ -41,10 +41,47 @@ def add_common_args(_parser): _parser.add_argument( "--service", "-s", - help="Service to use if none is configured (defaults to DEFAULT_SERVICE environment variable)", - default=config["DEFAULT_SERVICE"], + help="Service to use if none is configured (defaults to DEFAULT_SERVICE)", + default=config.get("DEFAULT_SERVICE"), type=str, ) + + default_temperature = config.get("DEFAULT_TEMPERATURE") or 0.7 + _parser.add_argument( + "--temperature", + "-t", + help="Temperature (defaults to DEFAULT_TEMPERATURE)", + default=default_temperature, + type=str, + ) + + default_max_tokens = config.get("DEFAULT_MAX_TOKENS") or 420 + _parser.add_argument( + "--max_tokens", + "-mt", + help="Max tokens to use (defaults to DEFAULT_MAX_TOKENS)", + default=default_max_tokens, + type=str, + ) + + default_top_p = config.get("DEFAULT_TOP_P") or -1 + _parser.add_argument( + "--top_p", + "-tp", + help="Sets a cumulative probability threshold for selecting candidate tokens, where only tokens with a cumulative probability higher than the threshold are considered, allowing for flexible control over the diversity of the generated output (defaults to DEFAULT_TOP_P).", + default=default_top_p, + type=str, + ) + + default_top_k = config.get("DEFAULT_TOP_K") or -1 + _parser.add_argument( + "--top_k", + "-tk", + help="Determines the number of top-scoring candidate tokens to consider at each decoding step, effectively limiting the diversity of the generated output (defaults to DEFAULT_TOP_K)", + default=default_top_k, + type=str, + ) + _parser.add_argument( "--quiet", "-q", diff --git a/prr/prompt/__init__.py b/prr/prompt/__init__.py index 511889e..056ed15 100644 --- a/prr/prompt/__init__.py +++ b/prr/prompt/__init__.py @@ -93,7 +93,7 @@ def __init__(self, path, args=None): # TODO/FIXME: should also include jinja includes self.dependency_files = [] - template_loader = jinja2.FileSystemLoader(searchpath=os.path.dirname(path)) + template_loader = jinja2.FileSystemLoader([os.path.dirname(path), "/"]) self.template_env = jinja2.Environment(loader=template_loader) root, extension = os.path.splitext(path) diff --git a/prr/prompt/prompt_template.py b/prr/prompt/prompt_template.py index 92cf4bd..ba29834 100644 --- a/prr/prompt/prompt_template.py +++ b/prr/prompt/prompt_template.py @@ -7,7 +7,11 @@ def __init__(self, content_template_string, search_path='.', role='user', name=N self.role = role self.name = name - template_loader = jinja2.FileSystemLoader(searchpath=self.search_path) + template_loader = jinja2.ChoiceLoader([ + jinja2.FileSystemLoader(search_path), + jinja2.FileSystemLoader(['/']), + ]) + template_env = jinja2.Environment(loader=template_loader) self.template = template_env.from_string(self.content_template_string) diff --git a/test/test_prompt_template.py b/test/test_prompt_template.py index 92c107a..c9856fc 100644 --- a/test/test_prompt_template.py +++ b/test/test_prompt_template.py @@ -65,8 +65,6 @@ def test_configured_messages_list_with_content_file(self): template = PromptTemplateMessages(yaml.safe_load(messages_config)) - remove_temp_file(temp_file_path) - assert template is not None @@ -83,4 +81,6 @@ def test_configured_messages_list_with_content_file(self): second_message = rendered_messages[1] assert second_message['content'] == 'Wollen Sie meine Kernel kompilieren?' assert second_message['role'] == 'user' - assert second_message['name'] == 'SuperUser' \ No newline at end of file + assert second_message['name'] == 'SuperUser' + + remove_temp_file(temp_file_path) From 99543ba91b1c5b49fce6a2a81efcd99cde643cc7 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Thu, 18 May 2023 18:13:39 +0000 Subject: [PATCH 09/16] wip --- prr/__main__.py | 10 ++++++---- prr/utils/run.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/prr/__main__.py b/prr/__main__.py index 4fbd9c2..863147d 100755 --- a/prr/__main__.py +++ b/prr/__main__.py @@ -9,6 +9,8 @@ from .utils.run import RunPromptCommand from .utils.watch import WatchPromptCommand +from .options import ModelOptions + config = load_config() @@ -46,7 +48,7 @@ def add_common_args(_parser): type=str, ) - default_temperature = config.get("DEFAULT_TEMPERATURE") or 0.7 + default_temperature = config.get("DEFAULT_TEMPERATURE") or ModelOptions.defaults['temperature'] _parser.add_argument( "--temperature", "-t", @@ -55,7 +57,7 @@ def add_common_args(_parser): type=str, ) - default_max_tokens = config.get("DEFAULT_MAX_TOKENS") or 420 + default_max_tokens = config.get("DEFAULT_MAX_TOKENS") or ModelOptions.defaults['max_tokens'] _parser.add_argument( "--max_tokens", "-mt", @@ -64,7 +66,7 @@ def add_common_args(_parser): type=str, ) - default_top_p = config.get("DEFAULT_TOP_P") or -1 + default_top_p = config.get("DEFAULT_TOP_P") or ModelOptions.defaults['top_p'] _parser.add_argument( "--top_p", "-tp", @@ -73,7 +75,7 @@ def add_common_args(_parser): type=str, ) - default_top_k = config.get("DEFAULT_TOP_K") or -1 + default_top_k = config.get("DEFAULT_TOP_K") or ModelOptions.defaults['top_k'] _parser.add_argument( "--top_k", "-tk", diff --git a/prr/utils/run.py b/prr/utils/run.py index bbc8d32..d691774 100755 --- a/prr/utils/run.py +++ b/prr/utils/run.py @@ -19,6 +19,7 @@ class RunPromptCommand: def __init__(self, args, prompt_args=None): self.args = args + print("--------args", args) self.prompt_args = prompt_args self.prompt_config = None From 7e43423b0c90086c9c9bf782671446cc90e6ff3c Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Fri, 19 May 2023 07:03:42 +0000 Subject: [PATCH 10/16] cmd line options + defaults from .prr_rc + config --- prr/__main__.py | 18 +++++------------- prr/options.py | 31 ++++++++++++++++++++++++++----- prr/runner.py | 8 +++++--- prr/service_config.py | 3 +++ prr/utils/run.py | 8 ++++++-- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/prr/__main__.py b/prr/__main__.py index 863147d..e1b7cef 100755 --- a/prr/__main__.py +++ b/prr/__main__.py @@ -13,7 +13,6 @@ config = load_config() - def main(): parser = argparse.ArgumentParser( description="Run a prompt against configured models.", @@ -40,6 +39,7 @@ def add_common_args(_parser): action="store_true", default=False, ) + _parser.add_argument( "--service", "-s", @@ -48,40 +48,32 @@ def add_common_args(_parser): type=str, ) - default_temperature = config.get("DEFAULT_TEMPERATURE") or ModelOptions.defaults['temperature'] _parser.add_argument( "--temperature", "-t", help="Temperature (defaults to DEFAULT_TEMPERATURE)", - default=default_temperature, - type=str, + type=float, ) - default_max_tokens = config.get("DEFAULT_MAX_TOKENS") or ModelOptions.defaults['max_tokens'] _parser.add_argument( "--max_tokens", "-mt", help="Max tokens to use (defaults to DEFAULT_MAX_TOKENS)", - default=default_max_tokens, - type=str, + type=int, ) - default_top_p = config.get("DEFAULT_TOP_P") or ModelOptions.defaults['top_p'] _parser.add_argument( "--top_p", "-tp", help="Sets a cumulative probability threshold for selecting candidate tokens, where only tokens with a cumulative probability higher than the threshold are considered, allowing for flexible control over the diversity of the generated output (defaults to DEFAULT_TOP_P).", - default=default_top_p, - type=str, + type=int, ) - default_top_k = config.get("DEFAULT_TOP_K") or ModelOptions.defaults['top_k'] _parser.add_argument( "--top_k", "-tk", help="Determines the number of top-scoring candidate tokens to consider at each decoding step, effectively limiting the diversity of the generated output (defaults to DEFAULT_TOP_K)", - default=default_top_k, - type=str, + type=int, ) _parser.add_argument( diff --git a/prr/options.py b/prr/options.py index 75ba274..2782fcc 100644 --- a/prr/options.py +++ b/prr/options.py @@ -1,7 +1,7 @@ +from prr.config import load_config + ALLOWED_OPTIONS = ["max_tokens", "temperature", "top_k", "top_p"] -# user-level defaults -# TODO/FIXME: make it user-configurable in ~/.prr* DEFAULT_OPTIONS = { "max_tokens": 4000, "temperature": 0.7, @@ -9,16 +9,19 @@ "top_p": -1 } -class ModelOptions: - defaults = DEFAULT_OPTIONS +config = load_config() +class ModelOptions: def __init__(self, options={}): + self.__init_defaults() + self.options_set = [] - self.update_options(ModelOptions.defaults) + self.update_options(self.defaults) self.update_options(options) def update_options(self, options): for key in options.keys(): + if options[key] != None: if key in ALLOWED_OPTIONS: if key not in self.options_set: self.options_set.append(key) @@ -41,3 +44,21 @@ def to_dict(self): dict[key] = self.option(key) return dict + + def __config_key_for_option_key(self, option_key): + return f"DEFAULT_{option_key.upper()}" + + def __init_defaults(self): + self.defaults = DEFAULT_OPTIONS.copy() + + for option_key in ALLOWED_OPTIONS: + config_key = self.__config_key_for_option_key(option_key) + defaults_value = config.get(config_key) + + if defaults_value: + if option_key == "temperature": + target_value = float(defaults_value) + else: + target_value = int(defaults_value) + + self.defaults[option_key] = target_value diff --git a/prr/runner.py b/prr/runner.py index a8b9461..e754d1b 100644 --- a/prr/runner.py +++ b/prr/runner.py @@ -11,9 +11,11 @@ def __init__(self, prompt_config): self.prompt_config = prompt_config self.saver = PromptRunSaver(self.prompt_config) - def run_service(self, service_name, save_run=False): + def run_service(self, service_name, service_options_overrides, save_run=False): service_config = self.prompt_config.service_with_name(service_name) + service_config.process_option_overrides(service_options_overrides) + service = service_registry.service_for_service_config(service_config) result = PromptRun(self.prompt_config, service, service_config).run() @@ -26,10 +28,10 @@ def run_service(self, service_name, save_run=False): return result, run_save_directory # runs all models defined for specified prompt - def run_all_configured_services(self): + def run_all_configured_services(self, service_options_overrides, save_run=False): results = {} for service_name in self.configured_services(): - results[service_name] = self.run_service(service_name) + results[service_name] = self.run_service(service_name, service_options_overrides, save_run) return results diff --git a/prr/service_config.py b/prr/service_config.py index 63782cc..d149163 100644 --- a/prr/service_config.py +++ b/prr/service_config.py @@ -6,6 +6,9 @@ def __init__(self, name, model, options=None): self.model = model # full model path, e.g. "openai/chat/gpt-5" self.options = ModelOptions(options or {}) + def process_option_overrides(self, option_overrides): + self.options.update_options(option_overrides) + # which model to use with the service # like gpt-4.5-turbo or claude-v1.3-100k def model_name(self): diff --git a/prr/utils/run.py b/prr/utils/run.py index d691774..75c4f64 100755 --- a/prr/utils/run.py +++ b/prr/utils/run.py @@ -71,7 +71,11 @@ def print_run_results(self, result, run_save_directory): self.console.log(f"💾 {run_save_directory}") def run_prompt_on_service(self, service_name, save=False): - options = self.prompt_config.options_for_service(service_name) + # TODO/FIXME: doing all this here just to get the actual options + # calculated after command line, defaults, config, etc + service_config = self.prompt_config.service_with_name(service_name) + service_config.process_option_overrides(self.args) + options = service_config.options with self.console.status( f":robot: [bold green]{service_name}[/bold green]" @@ -83,7 +87,7 @@ def run_prompt_on_service(self, service_name, save=False): status.update(status="running model", spinner="dots8Bit") - result, run_save_directory = self.runner.run_service(service_name, save) + result, run_save_directory = self.runner.run_service(service_name, self.args, save) self.print_run_results(result, run_save_directory) From 8fae552158c4c9d931ff8caaefa18d3aa7792ba1 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Fri, 19 May 2023 07:09:05 +0000 Subject: [PATCH 11/16] tests work again now --- prr/options.py | 15 ++++++++------- prr/prompt/prompt_loader.py | 2 +- prr/request.py | 4 +--- prr/utils/run.py | 1 - test/test_prompt_config.py | 8 ++++---- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/prr/options.py b/prr/options.py index 2782fcc..38adb6d 100644 --- a/prr/options.py +++ b/prr/options.py @@ -2,16 +2,17 @@ ALLOWED_OPTIONS = ["max_tokens", "temperature", "top_k", "top_p"] -DEFAULT_OPTIONS = { - "max_tokens": 4000, - "temperature": 0.7, - "top_k": -1, - "top_p": -1 -} config = load_config() class ModelOptions: + DEFAULT_OPTIONS = { + "max_tokens": 4000, + "temperature": 0.7, + "top_k": -1, + "top_p": -1 + } + def __init__(self, options={}): self.__init_defaults() @@ -49,7 +50,7 @@ def __config_key_for_option_key(self, option_key): return f"DEFAULT_{option_key.upper()}" def __init_defaults(self): - self.defaults = DEFAULT_OPTIONS.copy() + self.defaults = ModelOptions.DEFAULT_OPTIONS.copy() for option_key in ALLOWED_OPTIONS: config_key = self.__config_key_for_option_key(option_key) diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py index c89fe5e..4914d69 100644 --- a/prr/prompt/prompt_loader.py +++ b/prr/prompt/prompt_loader.py @@ -13,7 +13,7 @@ def __init__(self): def load_from_path(self, path): self.path = path - self.config = PromptConfig(self.__search_path()) + self.config = PromptConfig(self.__search_path(), os.path.basename(path)) self.__add_file_dependency(path) if self.__is_file_yaml(path): diff --git a/prr/request.py b/prr/request.py index 63d9b8d..700383e 100644 --- a/prr/request.py +++ b/prr/request.py @@ -9,10 +9,8 @@ def __init__(self, service_config, rendered_prompt_content): def to_dict(self): return { - "model": self.service_config.config_name(), + "model": self.service_config.model, "options": self.service_config.options.to_dict(), - # rendered prompt is saved to a separate file - # 'prompt_content': self.prompt_content, } def prompt_text(self, max_len=0): diff --git a/prr/utils/run.py b/prr/utils/run.py index 75c4f64..4576f03 100755 --- a/prr/utils/run.py +++ b/prr/utils/run.py @@ -19,7 +19,6 @@ class RunPromptCommand: def __init__(self, args, prompt_args=None): self.args = args - print("--------args", args) self.prompt_args = prompt_args self.prompt_config = None diff --git a/test/test_prompt_config.py b/test/test_prompt_config.py index a18ebc5..fcba822 100644 --- a/test/test_prompt_config.py +++ b/test/test_prompt_config.py @@ -62,10 +62,10 @@ def test_basic_services_model_list_no_options(self): assert services == ['openai/chat/gpt-4', 'anthropic/complete/claude-v1.3'] for service_name in services: - assert config.option_for_service(service_name, 'temperature') == ModelOptions.defaults['temperature'] - assert config.option_for_service(service_name, 'max_tokens') == ModelOptions.defaults['max_tokens'] - assert config.option_for_service(service_name, 'top_k') == ModelOptions.defaults['top_k'] - assert config.option_for_service(service_name, 'top_p') == ModelOptions.defaults['top_p'] + assert config.option_for_service(service_name, 'temperature') == ModelOptions.DEFAULT_OPTIONS['temperature'] + assert config.option_for_service(service_name, 'max_tokens') == ModelOptions.DEFAULT_OPTIONS['max_tokens'] + assert config.option_for_service(service_name, 'top_k') == ModelOptions.DEFAULT_OPTIONS['top_k'] + assert config.option_for_service(service_name, 'top_p') == ModelOptions.DEFAULT_OPTIONS['top_p'] def test_services(self): config = PromptConfig() From 5e69bd591ca983215b2f9e731b552760c354bf4c Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Fri, 19 May 2023 08:03:26 +0000 Subject: [PATCH 12/16] restructuring the code layout --- prr/__main__.py | 8 +- prr/{utils => commands}/run.py | 1 - prr/{utils => commands}/watch.py | 2 +- prr/prompt/__init__.py | 234 +------------------ prr/{options.py => prompt/model_options.py} | 2 +- prr/prompt/prompt_config.py | 2 +- prr/{ => prompt}/service_config.py | 2 +- prr/{runner.py => runner/__init__.py} | 2 +- prr/{ => runner}/prompt_run.py | 0 prr/{ => runner}/prompt_run_result.py | 0 prr/{ => runner}/request.py | 0 prr/{ => runner}/response.py | 0 prr/{ => runner}/saver.py | 0 prr/services/providers/anthropic/complete.py | 6 +- prr/services/providers/openai/chat.py | 6 +- prr/{ => services}/service_registry.py | 0 prr/{ => utils}/config.py | 0 test/{ => prompt}/helpers.py | 0 test/{ => prompt}/test_prompt.py | 3 +- test/{ => prompt}/test_prompt_config.py | 30 ++- test/{ => prompt}/test_prompt_loader.py | 3 +- test/{ => prompt}/test_prompt_template.py | 3 +- 22 files changed, 49 insertions(+), 255 deletions(-) rename prr/{utils => commands}/run.py (99%) rename prr/{utils => commands}/watch.py (98%) rename prr/{options.py => prompt/model_options.py} (97%) rename prr/{ => prompt}/service_config.py (95%) rename prr/{runner.py => runner/__init__.py} (95%) rename prr/{ => runner}/prompt_run.py (100%) rename prr/{ => runner}/prompt_run_result.py (100%) rename prr/{ => runner}/request.py (100%) rename prr/{ => runner}/response.py (100%) rename prr/{ => runner}/saver.py (100%) rename prr/{ => services}/service_registry.py (100%) rename prr/{ => utils}/config.py (100%) rename test/{ => prompt}/helpers.py (100%) rename test/{ => prompt}/test_prompt.py (79%) rename test/{ => prompt}/test_prompt_config.py (74%) rename test/{ => prompt}/test_prompt_loader.py (84%) rename test/{ => prompt}/test_prompt_template.py (96%) diff --git a/prr/__main__.py b/prr/__main__.py index e1b7cef..59986f3 100755 --- a/prr/__main__.py +++ b/prr/__main__.py @@ -4,12 +4,12 @@ import os import sys -from prr.config import load_config +from prr.utils.config import load_config -from .utils.run import RunPromptCommand -from .utils.watch import WatchPromptCommand +from .commands.run import RunPromptCommand +from .commands.watch import WatchPromptCommand -from .options import ModelOptions +from .prompt.model_options import ModelOptions config = load_config() diff --git a/prr/utils/run.py b/prr/commands/run.py similarity index 99% rename from prr/utils/run.py rename to prr/commands/run.py index 4576f03..70f794a 100755 --- a/prr/utils/run.py +++ b/prr/commands/run.py @@ -8,7 +8,6 @@ from rich.console import Console from rich.panel import Panel -# from prr.config import config from prr.prompt import Prompt from prr.runner import Runner diff --git a/prr/utils/watch.py b/prr/commands/watch.py similarity index 98% rename from prr/utils/watch.py rename to prr/commands/watch.py index 5b9457c..bf447b0 100755 --- a/prr/utils/watch.py +++ b/prr/commands/watch.py @@ -5,7 +5,7 @@ import time from prr.prompt import Prompt -from prr.utils.run import RunPromptCommand +from prr.commands.run import RunPromptCommand def timestamp_for_file(path): diff --git a/prr/prompt/__init__.py b/prr/prompt/__init__.py index 056ed15..9b23ae9 100644 --- a/prr/prompt/__init__.py +++ b/prr/prompt/__init__.py @@ -4,246 +4,14 @@ import yaml from jinja2 import meta -from ..service_config import ServiceConfig +from .service_config import ServiceConfig from .prompt_config import PromptConfig from .prompt_template import PromptTemplate -# parse something like: -# -# services: -# gpt35crazy: -# model: 'openai/chat/gpt-3.5-turbo' -# options: -# temperature: 0.99 -# claudev1smart: -# model: 'anthropic/complete/claude-v1' -# options: -# temperature: 0 -# options: -# temperature: 0.7 -# max_tokens: 64 -def parse_specifically_configured_services(services_config): - options = services_config.get("options") - services_config.pop("options") - - service_names = services_config.keys() - - services = {} - - for service_name in service_names: - service_config = services_config[service_name] - model = service_config["model"] - service_config.pop("model") - - merged_options = options.copy() - - if service_config["options"]: - service_options = service_config["options"] - merged_options.update(service_options) - - services[service_name] = ServiceConfig(model, merged_options) - - return services - - -# parse something like: -# -# services: -# models: -# - 'openai/chat/gpt-3.5-turbo' -# - 'anthropic/complete/claude-v1' -# options: -# temperature: 0.7 -# max_tokens: 100 -# top_p: 1.0 -# top_k: 40 -def parse_generally_configured_services(services_config): - options = services_config.get("options") - models = services_config.get("models") - - services = {} - - for model in models: - services[model] = ServiceConfig(model, options) - - return services - - -def parse_config_into_services(services_config): - if services_config.get("models"): - return parse_generally_configured_services(services_config) - else: - return parse_specifically_configured_services(services_config) - - class Prompt: def __init__(self, content, config=None, args=None): self.content = content self.config = config self.args = args -class PromptOLD: - def __init__(self, path, args=None): - self.path = None - self.messages = None - self.template = None - self.services = {} - self.args = args - # TODO/FIXME: should also include jinja includes - self.dependency_files = [] - - template_loader = jinja2.FileSystemLoader([os.path.dirname(path), "/"]) - self.template_env = jinja2.Environment(loader=template_loader) - - root, extension = os.path.splitext(path) - - if extension == ".yaml": - self.load_yaml_file(path) - else: - self.load_text_file(path) - - def configured_service_names(self): - if self.services: - return list(self.services.keys()) - - return [] - - def parse_messages(self, messages): - # expand content_file field in messages - self.messages = [] - self.dependency_files = [] - root_path = os.path.dirname(self.path) - - if messages: - for message in messages: - if message.get("content_file"): - updated_message = message.copy() - file_path = os.path.join( - root_path, updated_message.pop("content_file") - ) - - with open(file_path, "r") as f: - updated_message.update({"content": f.read()}) - self.messages.append(updated_message) - self.dependency_files.append(file_path) - else: - self.messages.append(message) - - def add_dependency_files_from_jinja_template(self, jinja_template_content): - parsed_content = self.template_env.parse(jinja_template_content) - referenced_templates = meta.find_referenced_templates(parsed_content) - - for referenced_template in referenced_templates: - template_path = os.path.join( - os.path.dirname(self.path), referenced_template - ) - self.dependency_files.append(template_path) - - def parse_services(self, services_config): - self.services = parse_config_into_services(services_config) - - def parse_prompt_config(self, prompt_config): - if prompt_config.get("messages"): - self.parse_messages(prompt_config["messages"]) - elif prompt_config.get("content"): - self.template = self.load_jinja_template_from_string( - prompt_config["content"] - ) - elif prompt_config.get("content_file"): - self.template = self.load_jinja_template_from_file( - prompt_config["content_file"] - ) - - def config_for_service(self, service_name): - if self.services: - if self.services.get(service_name): - return self.services[service_name] - - return ServiceConfig(service_name) - - def load_yaml_file(self, path): - with open(path, "r") as stream: - try: - file_contents = self.deal_with_shebang_line(stream) - data = yaml.safe_load(file_contents) - self.path = path - except yaml.YAMLError as exc: - print(exc) - - if data: - if data["services"]: - self.parse_services(data["services"]) - if data["prompt"]: - self.parse_prompt_config(data["prompt"]) - - def deal_with_shebang_line(self, stream): - file_contents = stream.readlines() - - # if the file starts with a shebang, like #!/usr/bin/prr, omit it - if file_contents[0].startswith("#!/"): - if file_contents[1] == "\n": - # allow for one empty line below the shebang - file_contents = file_contents[2:] - else: - file_contents = file_contents[1:] - - return "".join(file_contents) - - def load_jinja_template_from_string(self, content): - self.add_dependency_files_from_jinja_template(content) - return self.template_env.from_string(content) - - def load_jinja_template_from_file(self, template_subpath): - try: - with open( - os.path.join(os.path.dirname(self.path), template_subpath), "r" - ) as stream: - self.add_dependency_files_from_jinja_template(stream.read()) - - return self.template_env.get_template(template_subpath) - except FileNotFoundError: - print(f"Could not find template file: {template_subpath}") - exit(-1) - - def load_text_file(self, path): - self.path = path - - with open(path, "r") as stream: - file_contents = self.deal_with_shebang_line(stream) - self.template = self.template_env.from_string(file_contents) - - def message_text_description(self, message): - name = message.get("name") - role = message.get("role") - content = message.get("content") - - if name: - return f"{name} ({role}): {content}" - else: - return f"{role}: {content}" - - def text(self): - if self.messages: - return "\n".join( - [self.message_text_description(msg) for msg in self.messages] - ) - - if self.args: - return self.template.render({"prompt_args": self.args}) - - return self.template.render() - - def text_len(self): - return len(self.text()) - - def dump(self): - return yaml.dump({"text": self.text(), "messages": self.messages}) - - def text_abbrev(self, max_len=25): - if self.text_len() > max_len: - str = self.text()[0:max_len] + "..." - else: - str = self.text() - - return str.replace("\n", " ").replace(" ", " ") diff --git a/prr/options.py b/prr/prompt/model_options.py similarity index 97% rename from prr/options.py rename to prr/prompt/model_options.py index 38adb6d..313838e 100644 --- a/prr/options.py +++ b/prr/prompt/model_options.py @@ -1,4 +1,4 @@ -from prr.config import load_config +from prr.utils.config import load_config ALLOWED_OPTIONS = ["max_tokens", "temperature", "top_k", "top_p"] diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py index cb1ea19..f0f9cc5 100644 --- a/prr/prompt/prompt_config.py +++ b/prr/prompt/prompt_config.py @@ -1,6 +1,6 @@ import yaml -from ..service_config import ServiceConfig +from .service_config import ServiceConfig from .prompt_template import PromptTemplateSimple, PromptTemplateMessages class PromptConfig: diff --git a/prr/service_config.py b/prr/prompt/service_config.py similarity index 95% rename from prr/service_config.py rename to prr/prompt/service_config.py index d149163..d6ab276 100644 --- a/prr/service_config.py +++ b/prr/prompt/service_config.py @@ -1,4 +1,4 @@ -from .options import ModelOptions +from .model_options import ModelOptions class ServiceConfig: def __init__(self, name, model, options=None): diff --git a/prr/runner.py b/prr/runner/__init__.py similarity index 95% rename from prr/runner.py rename to prr/runner/__init__.py index e754d1b..d6757be 100644 --- a/prr/runner.py +++ b/prr/runner/__init__.py @@ -1,6 +1,6 @@ from .prompt_run import PromptRun from .saver import PromptRunSaver -from .service_registry import ServiceRegistry +from ..services.service_registry import ServiceRegistry service_registry = ServiceRegistry() service_registry.register_all_services() diff --git a/prr/prompt_run.py b/prr/runner/prompt_run.py similarity index 100% rename from prr/prompt_run.py rename to prr/runner/prompt_run.py diff --git a/prr/prompt_run_result.py b/prr/runner/prompt_run_result.py similarity index 100% rename from prr/prompt_run_result.py rename to prr/runner/prompt_run_result.py diff --git a/prr/request.py b/prr/runner/request.py similarity index 100% rename from prr/request.py rename to prr/runner/request.py diff --git a/prr/response.py b/prr/runner/response.py similarity index 100% rename from prr/response.py rename to prr/runner/response.py diff --git a/prr/saver.py b/prr/runner/saver.py similarity index 100% rename from prr/saver.py rename to prr/runner/saver.py diff --git a/prr/services/providers/anthropic/complete.py b/prr/services/providers/anthropic/complete.py index a45450f..0152c12 100644 --- a/prr/services/providers/anthropic/complete.py +++ b/prr/services/providers/anthropic/complete.py @@ -5,9 +5,9 @@ import anthropic -from prr.config import load_config -from prr.request import ServiceRequest -from prr.response import ServiceResponse +from prr.utils.config import load_config +from prr.runner.request import ServiceRequest +from prr.runner.response import ServiceResponse config = load_config() diff --git a/prr/services/providers/openai/chat.py b/prr/services/providers/openai/chat.py index a783982..a181511 100644 --- a/prr/services/providers/openai/chat.py +++ b/prr/services/providers/openai/chat.py @@ -1,8 +1,8 @@ import openai -from prr.config import load_config -from prr.request import ServiceRequest -from prr.response import ServiceResponse +from prr.utils.config import load_config +from prr.runner.request import ServiceRequest +from prr.runner.response import ServiceResponse config = load_config() openai.api_key = config["OPENAI_API_KEY"] diff --git a/prr/service_registry.py b/prr/services/service_registry.py similarity index 100% rename from prr/service_registry.py rename to prr/services/service_registry.py diff --git a/prr/config.py b/prr/utils/config.py similarity index 100% rename from prr/config.py rename to prr/utils/config.py diff --git a/test/helpers.py b/test/prompt/helpers.py similarity index 100% rename from test/helpers.py rename to test/prompt/helpers.py diff --git a/test/test_prompt.py b/test/prompt/test_prompt.py similarity index 79% rename from test/test_prompt.py rename to test/prompt/test_prompt.py index af74279..39828db 100644 --- a/test/test_prompt.py +++ b/test/prompt/test_prompt.py @@ -1,6 +1,7 @@ import os import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) from prr.prompt import Prompt diff --git a/test/test_prompt_config.py b/test/prompt/test_prompt_config.py similarity index 74% rename from test/test_prompt_config.py rename to test/prompt/test_prompt_config.py index fcba822..cda756c 100644 --- a/test/test_prompt_config.py +++ b/test/prompt/test_prompt_config.py @@ -1,9 +1,9 @@ import os import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) from prr.prompt.prompt_config import PromptConfig -from prr.options import ModelOptions +from prr.prompt.model_options import ModelOptions class TestPromptConfig: @@ -45,7 +45,31 @@ def test_basic_services_model_list(self): assert config.option_for_service(service_name, 'temperature') == 0.42 assert config.option_for_service(service_name, 'max_tokens') == 1337 - def test_basic_services_model_list_no_options(self): + def test_basic_services_model_list_with_some_options(self): + config = PromptConfig() + config.load_from_config_contents(""" +prompt: + content: 'foo bar' +services: + models: + - 'openai/chat/gpt-4' + - 'anthropic/complete/claude-v1.3' + options: + top_k: -1.337 +""") + + assert config is not None + services = config.configured_services() + + assert services == ['openai/chat/gpt-4', 'anthropic/complete/claude-v1.3'] + + for service_name in services: + assert config.option_for_service(service_name, 'temperature') == ModelOptions.DEFAULT_OPTIONS['temperature'] + assert config.option_for_service(service_name, 'max_tokens') == ModelOptions.DEFAULT_OPTIONS['max_tokens'] + assert config.option_for_service(service_name, 'top_k') == -1.337 + assert config.option_for_service(service_name, 'top_p') == ModelOptions.DEFAULT_OPTIONS['top_p'] + + def test_basic_services_model_list_with_no_options(self): config = PromptConfig() config.load_from_config_contents(""" prompt: diff --git a/test/test_prompt_loader.py b/test/prompt/test_prompt_loader.py similarity index 84% rename from test/test_prompt_loader.py rename to test/prompt/test_prompt_loader.py index e5f3d8b..48c1892 100644 --- a/test/test_prompt_loader.py +++ b/test/prompt/test_prompt_loader.py @@ -1,6 +1,7 @@ import os import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) from prr.prompt.prompt_loader import PromptConfigLoader diff --git a/test/test_prompt_template.py b/test/prompt/test_prompt_template.py similarity index 96% rename from test/test_prompt_template.py rename to test/prompt/test_prompt_template.py index c9856fc..a8358d1 100644 --- a/test/test_prompt_template.py +++ b/test/prompt/test_prompt_template.py @@ -2,7 +2,8 @@ import sys import yaml -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) from prr.prompt.prompt_template import PromptTemplateSimple, PromptTemplateMessages from helpers import create_temp_file, remove_temp_file From 6e3a812aedc9941e5a3faff14a11b9e30c7e57a2 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Fri, 19 May 2023 08:44:04 +0000 Subject: [PATCH 13/16] dependencies watching works again --- .env.example | 13 +++++---- .../{html_boilerplate => _html_boilerplate} | 0 examples/code/html_boilerplate.yaml | 2 +- examples/shebang/write_tests | 5 ++++ prr/commands/watch.py | 8 +++--- prr/prompt/prompt_config.py | 9 +++++++ prr/prompt/prompt_loader.py | 23 +++++++++++----- prr/prompt/prompt_template.py | 27 ++++++++++++++++--- 8 files changed, 69 insertions(+), 18 deletions(-) rename examples/code/{html_boilerplate => _html_boilerplate} (100%) create mode 100755 examples/shebang/write_tests diff --git a/.env.example b/.env.example index a7f8521..a5b1d53 100644 --- a/.env.example +++ b/.env.example @@ -7,8 +7,11 @@ ANTHROPIC_API_KEY="sk-ant-..." # when things aren't specify, these defaults will kick in: #DEFAULT_SERVICE="anthropic/complete/claude-v1.3-100k" #DEFAULT_SERVICE="openai/chat/gpt-4" -DEFAULT_SERVICE="openai/chat/gpt-3.5-turbo" -DEFAULT_TEMPERATURE=0.6 -DEFAULT_MAX_TOKENS=1337 -DEFAULT_TOP_K=-1 -DEFAULT_TOP_P=-1 \ No newline at end of file +#DEFAULT_SERVICE="openai/chat/gpt-3.5-turbo" + +# WARNING/TODO/FIXME: do not specify below options when running pytest +# +#DEFAULT_TEMPERATURE=0.6 +#DEFAULT_MAX_TOKENS=1337 +#DEFAULT_TOP_K=-1 +#DEFAULT_TOP_P=-1 diff --git a/examples/code/html_boilerplate b/examples/code/_html_boilerplate similarity index 100% rename from examples/code/html_boilerplate rename to examples/code/_html_boilerplate diff --git a/examples/code/html_boilerplate.yaml b/examples/code/html_boilerplate.yaml index e9fe6ff..74aa653 100644 --- a/examples/code/html_boilerplate.yaml +++ b/examples/code/html_boilerplate.yaml @@ -1,6 +1,6 @@ version: 1 prompt: - content_file: 'html_boilerplate' + content_file: '_html_boilerplate' services: gpt4_temp7: model: 'openai/chat/gpt-4' diff --git a/examples/shebang/write_tests b/examples/shebang/write_tests new file mode 100755 index 0000000..44d3ffd --- /dev/null +++ b/examples/shebang/write_tests @@ -0,0 +1,5 @@ +#!/workspaces/prr/prr script + +Write tests using pytest for the code below: + +{% include prompt_args %} diff --git a/prr/commands/watch.py b/prr/commands/watch.py index bf447b0..1376717 100755 --- a/prr/commands/watch.py +++ b/prr/commands/watch.py @@ -7,6 +7,7 @@ from prr.prompt import Prompt from prr.commands.run import RunPromptCommand +from prr.prompt.prompt_loader import PromptConfigLoader def timestamp_for_file(path): if os.path.exists(path): @@ -34,9 +35,10 @@ def update_timestamps(self, ready_timestamps=None): self.file_timestamps = self.current_timestamps() def setup_files_to_monitor(self): - prompt = Prompt(self.args["prompt_path"], self.prompt_args) - self.files = [prompt.path] - self.files.extend(prompt.dependency_files) + loader = PromptConfigLoader() + prompt_config = loader.load_from_path(self.args["prompt_path"]) + + self.files = loader.file_dependencies self.update_timestamps() def files_changed(self): diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py index f0f9cc5..1af5543 100644 --- a/prr/prompt/prompt_config.py +++ b/prr/prompt/prompt_config.py @@ -73,6 +73,15 @@ def options_for_service(self, service_name): def option_for_service(self, service_name, option_name): return self.options_for_service(service_name).option(option_name) + + def file_dependencies(self): + _dependencies = [] + for message in self.template.messages: + for dependency in message.file_dependencies: + if dependency not in _dependencies: + _dependencies.append(dependency) + + return _dependencies #################################################### diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py index 4914d69..78deb25 100644 --- a/prr/prompt/prompt_loader.py +++ b/prr/prompt/prompt_loader.py @@ -1,5 +1,4 @@ import os - import yaml from . import Prompt @@ -8,13 +7,12 @@ class PromptConfigLoader: def __init__(self): - self.dependency_files = [] + self.file_dependencies = [] self.config = None def load_from_path(self, path): self.path = path self.config = PromptConfig(self.__search_path(), os.path.basename(path)) - self.__add_file_dependency(path) if self.__is_file_yaml(path): # prompt is in yaml config file format @@ -22,6 +20,8 @@ def load_from_path(self, path): else: # simple text (or jinja) file, no config self.__load_text_file(path) + + self.__add_file_dependencies() return self.config @@ -49,6 +49,17 @@ def __load_yaml_file(self, path): except yaml.YAMLError as exc: print(exc) - def __add_file_dependency(self, path): - if not path in self.dependency_files: - self.dependency_files.append(path) + def __add_file_dependencies(self): + self.__add_file_dependency(self.path) + + for file_dependency in self.config.file_dependencies(): + self.__add_file_dependency(file_dependency) + + def __add_file_dependency(self, file_path): + if os.path.isabs(file_path): + absolute_path = file_path + else: + absolute_path = os.path.join(os.path.dirname(self.path), os.path.basename(file_path)) + + if not absolute_path in self.file_dependencies: + self.file_dependencies.append(absolute_path) \ No newline at end of file diff --git a/prr/prompt/prompt_template.py b/prr/prompt/prompt_template.py index ba29834..b2436c2 100644 --- a/prr/prompt/prompt_template.py +++ b/prr/prompt/prompt_template.py @@ -1,3 +1,4 @@ +import os import jinja2 class PromptMessage: @@ -6,14 +7,18 @@ def __init__(self, content_template_string, search_path='.', role='user', name=N self.search_path = search_path self.role = role self.name = name - + self.file_dependencies = [] + template_loader = jinja2.ChoiceLoader([ jinja2.FileSystemLoader(search_path), jinja2.FileSystemLoader(['/']), ]) - template_env = jinja2.Environment(loader=template_loader) - self.template = template_env.from_string(self.content_template_string) + self.template_env = jinja2.Environment(loader=template_loader) + self.__add_dependency_files_from_jinja_template(content_template_string) + + self.template = self.template_env.from_string(self.content_template_string) + def render_text(self, args=[]): return self.template.render({"prompt_args": args}) @@ -29,6 +34,13 @@ def render_message(self, args=[]): return _message + def __add_dependency_files_from_jinja_template(self, jinja_template_content): + parsed_content = self.template_env.parse(jinja_template_content) + referenced_templates = jinja2.meta.find_referenced_templates(parsed_content) + + self.file_dependencies.extend(referenced_templates) + + # base class class PromptTemplate: def __init__(self): @@ -42,6 +54,15 @@ def render_text(self, args=[]): def render_messages(self, args=[]): return [message.render_message(args) for message in self.messages] + def file_dependencies(self): + _dependencies = [] + for message in self.messages: + for dependency in message.file_dependencies: + if dependency not in _dependencies: + _dependencies.append(dependency) + + return _dependencies + # just a text/template file or prompt.contents from config class PromptTemplateSimple(PromptTemplate): def __init__(self, template_string, search_path='.'): From a35e4af726e1b4e9906d12892268ef0cc7602aad Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Fri, 19 May 2023 10:12:38 +0000 Subject: [PATCH 14/16] formatting --- prr/__main__.py | 2 +- prr/commands/run.py | 8 +- prr/commands/watch.py | 4 +- prr/prompt/__init__.py | 5 +- prr/prompt/model_options.py | 50 ++- prr/prompt/prompt_config.py | 346 ++++++++++--------- prr/prompt/prompt_loader.py | 124 +++---- prr/prompt/prompt_template.py | 136 ++++---- prr/prompt/service_config.py | 36 +- prr/runner/__init__.py | 7 +- prr/runner/saver.py | 5 +- prr/services/providers/anthropic/complete.py | 3 +- prr/services/providers/openai/chat.py | 4 +- prr/services/service_registry.py | 1 + test/prompt/helpers.py | 21 +- test/prompt/test_prompt.py | 7 +- test/prompt/test_prompt_config.py | 175 ++++++---- test/prompt/test_prompt_loader.py | 19 +- test/prompt/test_prompt_template.py | 102 +++--- 19 files changed, 557 insertions(+), 498 deletions(-) diff --git a/prr/__main__.py b/prr/__main__.py index 59986f3..d27e727 100755 --- a/prr/__main__.py +++ b/prr/__main__.py @@ -8,11 +8,11 @@ from .commands.run import RunPromptCommand from .commands.watch import WatchPromptCommand - from .prompt.model_options import ModelOptions config = load_config() + def main(): parser = argparse.ArgumentParser( description="Run a prompt against configured models.", diff --git a/prr/commands/run.py b/prr/commands/run.py index 70f794a..e3228c4 100755 --- a/prr/commands/run.py +++ b/prr/commands/run.py @@ -9,12 +9,12 @@ from rich.panel import Panel from prr.prompt import Prompt -from prr.runner import Runner - from prr.prompt.prompt_loader import PromptConfigLoader +from prr.runner import Runner console = Console(log_time=False, log_path=False) + class RunPromptCommand: def __init__(self, args, prompt_args=None): self.args = args @@ -85,7 +85,9 @@ def run_prompt_on_service(self, service_name, save=False): status.update(status="running model", spinner="dots8Bit") - result, run_save_directory = self.runner.run_service(service_name, self.args, save) + result, run_save_directory = self.runner.run_service( + service_name, self.args, save + ) self.print_run_results(result, run_save_directory) diff --git a/prr/commands/watch.py b/prr/commands/watch.py index 1376717..b108290 100755 --- a/prr/commands/watch.py +++ b/prr/commands/watch.py @@ -4,11 +4,11 @@ import os import time -from prr.prompt import Prompt from prr.commands.run import RunPromptCommand - +from prr.prompt import Prompt from prr.prompt.prompt_loader import PromptConfigLoader + def timestamp_for_file(path): if os.path.exists(path): return os.path.getmtime(path) diff --git a/prr/prompt/__init__.py b/prr/prompt/__init__.py index 9b23ae9..8711912 100644 --- a/prr/prompt/__init__.py +++ b/prr/prompt/__init__.py @@ -4,14 +4,13 @@ import yaml from jinja2 import meta -from .service_config import ServiceConfig - from .prompt_config import PromptConfig from .prompt_template import PromptTemplate +from .service_config import ServiceConfig + class Prompt: def __init__(self, content, config=None, args=None): self.content = content self.config = config self.args = args - diff --git a/prr/prompt/model_options.py b/prr/prompt/model_options.py index 313838e..6c551ef 100644 --- a/prr/prompt/model_options.py +++ b/prr/prompt/model_options.py @@ -5,13 +5,9 @@ config = load_config() + class ModelOptions: - DEFAULT_OPTIONS = { - "max_tokens": 4000, - "temperature": 0.7, - "top_k": -1, - "top_p": -1 - } + DEFAULT_OPTIONS = {"max_tokens": 4000, "temperature": 0.7, "top_k": -1, "top_p": -1} def __init__(self, options={}): self.__init_defaults() @@ -22,12 +18,12 @@ def __init__(self, options={}): def update_options(self, options): for key in options.keys(): - if options[key] != None: - if key in ALLOWED_OPTIONS: - if key not in self.options_set: - self.options_set.append(key) - - setattr(self, key, options[key]) + if options[key] != None: + if key in ALLOWED_OPTIONS: + if key not in self.options_set: + self.options_set.append(key) + + setattr(self, key, options[key]) def description(self): return " ".join([f"{key}={self.option(key)}" for key in self.options_set]) @@ -47,19 +43,19 @@ def to_dict(self): return dict def __config_key_for_option_key(self, option_key): - return f"DEFAULT_{option_key.upper()}" - + return f"DEFAULT_{option_key.upper()}" + def __init_defaults(self): - self.defaults = ModelOptions.DEFAULT_OPTIONS.copy() - - for option_key in ALLOWED_OPTIONS: - config_key = self.__config_key_for_option_key(option_key) - defaults_value = config.get(config_key) - - if defaults_value: - if option_key == "temperature": - target_value = float(defaults_value) - else: - target_value = int(defaults_value) - - self.defaults[option_key] = target_value + self.defaults = ModelOptions.DEFAULT_OPTIONS.copy() + + for option_key in ALLOWED_OPTIONS: + config_key = self.__config_key_for_option_key(option_key) + defaults_value = config.get(config_key) + + if defaults_value: + if option_key == "temperature": + target_value = float(defaults_value) + else: + target_value = int(defaults_value) + + self.defaults[option_key] = target_value diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py index 1af5543..d92e49e 100644 --- a/prr/prompt/prompt_config.py +++ b/prr/prompt/prompt_config.py @@ -1,180 +1,184 @@ import yaml +from .prompt_template import PromptTemplateMessages, PromptTemplateSimple from .service_config import ServiceConfig -from .prompt_template import PromptTemplateSimple, PromptTemplateMessages + class PromptConfig: - # raw_config_content is text to be parsed into YAML - def __init__(self, search_path='.', filename=None): - # where are we supposed to look for referenced files - self.search_path = search_path + # raw_config_content is text to be parsed into YAML + def __init__(self, search_path=".", filename=None): + # where are we supposed to look for referenced files + self.search_path = search_path + + # "foo" or "foo.yaml" - no path + self.filename = filename + + # template: (PromptTemplate) + self.template = None + + # services: (ServiceConfig) + self.services = {} + + # version: 1 + self.version = None + + def template_text(self): + return self.template.render_text() + + # raw YAML file + def load_from_config_contents(self, raw_config_content): + # raw YAML string + self.raw_config_content = raw_config_content + + # parse raw YAML content into a dictionary + self.__parse_raw_config() + + # parse that dictionary into respective parts of prompt config + self.__parse() + + # raw prompt template file + def load_from_template_contents(self, raw_template_content): + self.__parse_prompt_template_simple(raw_template_content) + + # raw prompt template file from file + def load_from_template_contents_at_path(self, path): + try: + with open(path, "r") as file: + return self.__parse_prompt_template_simple(file.read()) + + except FileNotFoundError: + print("The specified file does not exist.") + + except PermissionError: + print("You do not have permission to access the specified file.") + + except Exception as e: + print("An error occurred while opening the file:", str(e)) - # "foo" or "foo.yaml" - no path - self.filename = filename - - # template: (PromptTemplate) - self.template = None + # list keys/names of all services that we have configured in the config file + def configured_services(self): + return list(self.services.keys()) - # services: (ServiceConfig) - self.services = {} - - # version: 1 - self.version = None - - def template_text(self): - return self.template.render_text() - - # raw YAML file - def load_from_config_contents(self, raw_config_content): - # raw YAML string - self.raw_config_content = raw_config_content - - # parse raw YAML content into a dictionary - self.__parse_raw_config() - - # parse that dictionary into respective parts of prompt config - self.__parse() - - # raw prompt template file - def load_from_template_contents(self, raw_template_content): - self.__parse_prompt_template_simple(raw_template_content) - - # raw prompt template file from file - def load_from_template_contents_at_path(self, path): - try: - with open(path, "r") as file: - return self.__parse_prompt_template_simple(file.read()) - - except FileNotFoundError: - print("The specified file does not exist.") - - except PermissionError: - print("You do not have permission to access the specified file.") - - except Exception as e: - print("An error occurred while opening the file:", str(e)) - - # list keys/names of all services that we have configured in the config file - def configured_services(self): - return list(self.services.keys()) - - def service_with_name(self, service_name): - service_config = self.services.get(service_name) - - if service_config: - return service_config - else: - return ServiceConfig(service_name, service_name) - - - # returns options for specific service, already includes all option inheritance - def options_for_service(self, service_name): - return self.service_with_name(service_name).options - - def option_for_service(self, service_name, option_name): - return self.options_for_service(service_name).option(option_name) - - def file_dependencies(self): - _dependencies = [] - for message in self.template.messages: - for dependency in message.file_dependencies: - if dependency not in _dependencies: - _dependencies.append(dependency) - - return _dependencies - - #################################################### - - def __parse(self): - self.__parse_version() - self.__parse_prompt() - self.__parse_services() - - def __parse_raw_config(self): - try: - self.config_content = yaml.safe_load(self.raw_config_content) - except yaml.YAMLError as exc: - print(exc) - - def __parse_version(self): - if self.config_content: - self.version = self.config_content.get('version') - - def __parse_prompt_template_simple(self, content): - self.template = PromptTemplateSimple(content, self.search_path) - - # high level "prompt:" parsing - def __parse_prompt(self): - if self.config_content: - prompt = self.config_content.get('prompt') - - if prompt: - content_file = prompt.get('content_file') - content = prompt.get('content') - messages = prompt.get('messages') - - if content_file: - include_contents = "{% include '" + content_file + "' %}" - self.template = PromptTemplateSimple(include_contents, self.search_path) - elif content: - self.template = PromptTemplateSimple(content, self.search_path) - elif messages: - self.template = PromptTemplateMessages(messages, self.search_path) - - # high level "services:" parsing - def __parse_services(self): - if self.config_content: - _services = self.config_content.get('services') - - if _services: - options_for_all_services = _services.get('options') - - # - # if we have models + prompt-level model options - # - # services: - # models: - # - 'openai/chat/gpt-4' - # - 'anthropic/complete/claude-v1.3-100k' - # options: - # max_tokens: 1337 - _models = _services.get('models') - if _models: - for _model_name in _models: - service_config = ServiceConfig(_model_name, - _model_name, - options_for_all_services) - - self.services[_model_name] = service_config + def service_with_name(self, service_name): + service_config = self.services.get(service_name) + if service_config: + return service_config else: - # - # if we have services defined with options for each - # - # services: - # mygpt4: - # model: 'openai/chat/gpt-4' - # options: - # temperature: 0.2 - # max_tokens: 4000 - # options: - # max_tokens: 1337 - for _service_name in _services: - if _service_name not in ['options', 'models']: - service = _services[_service_name] - - # start with options for all services - # defined on a higher level - options = options_for_all_services.copy() - - # update with service-level options - service_level_options = service.get('options') - - if service_level_options: - options.update(service_level_options) - - model = service.get('model') - - service_config = ServiceConfig(_service_name, model, options) - - self.services[_service_name] = service_config + return ServiceConfig(service_name, service_name) + + # returns options for specific service, already includes all option inheritance + def options_for_service(self, service_name): + return self.service_with_name(service_name).options + + def option_for_service(self, service_name, option_name): + return self.options_for_service(service_name).option(option_name) + + def file_dependencies(self): + _dependencies = [] + for message in self.template.messages: + for dependency in message.file_dependencies: + if dependency not in _dependencies: + _dependencies.append(dependency) + + return _dependencies + + #################################################### + + def __parse(self): + self.__parse_version() + self.__parse_prompt() + self.__parse_services() + + def __parse_raw_config(self): + try: + self.config_content = yaml.safe_load(self.raw_config_content) + except yaml.YAMLError as exc: + print(exc) + + def __parse_version(self): + if self.config_content: + self.version = self.config_content.get("version") + + def __parse_prompt_template_simple(self, content): + self.template = PromptTemplateSimple(content, self.search_path) + + # high level "prompt:" parsing + def __parse_prompt(self): + if self.config_content: + prompt = self.config_content.get("prompt") + + if prompt: + content_file = prompt.get("content_file") + content = prompt.get("content") + messages = prompt.get("messages") + + if content_file: + include_contents = "{% include '" + content_file + "' %}" + self.template = PromptTemplateSimple( + include_contents, self.search_path + ) + elif content: + self.template = PromptTemplateSimple(content, self.search_path) + elif messages: + self.template = PromptTemplateMessages(messages, self.search_path) + + # high level "services:" parsing + def __parse_services(self): + if self.config_content: + _services = self.config_content.get("services") + + if _services: + options_for_all_services = _services.get("options") + + # + # if we have models + prompt-level model options + # + # services: + # models: + # - 'openai/chat/gpt-4' + # - 'anthropic/complete/claude-v1.3-100k' + # options: + # max_tokens: 1337 + _models = _services.get("models") + if _models: + for _model_name in _models: + service_config = ServiceConfig( + _model_name, _model_name, options_for_all_services + ) + + self.services[_model_name] = service_config + + else: + # + # if we have services defined with options for each + # + # services: + # mygpt4: + # model: 'openai/chat/gpt-4' + # options: + # temperature: 0.2 + # max_tokens: 4000 + # options: + # max_tokens: 1337 + for _service_name in _services: + if _service_name not in ["options", "models"]: + service = _services[_service_name] + + # start with options for all services + # defined on a higher level + options = options_for_all_services.copy() + + # update with service-level options + service_level_options = service.get("options") + + if service_level_options: + options.update(service_level_options) + + model = service.get("model") + + service_config = ServiceConfig( + _service_name, model, options + ) + + self.services[_service_name] = service_config diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py index 78deb25..7325f95 100644 --- a/prr/prompt/prompt_loader.py +++ b/prr/prompt/prompt_loader.py @@ -1,65 +1,73 @@ import os + import yaml from . import Prompt -from .prompt_template import PromptTemplate, PromptTemplateSimple, PromptTemplateMessages from .prompt_config import PromptConfig +from .prompt_template import ( + PromptTemplate, + PromptTemplateMessages, + PromptTemplateSimple, +) + class PromptConfigLoader: - def __init__(self): - self.file_dependencies = [] - self.config = None - - def load_from_path(self, path): - self.path = path - self.config = PromptConfig(self.__search_path(), os.path.basename(path)) - - if self.__is_file_yaml(path): - # prompt is in yaml config file format - self.__load_yaml_file(path) - else: - # simple text (or jinja) file, no config - self.__load_text_file(path) - - self.__add_file_dependencies() - - return self.config - - ##################################### - - def __is_file_yaml(self, path): - root, extension = os.path.splitext(path) - - if extension == ".yaml": - return True - - return False - - def __search_path(self): - return os.path.dirname(self.path) - - def __load_text_file(self, path): - self.config.load_from_template_contents_at_path(path) - - def __load_yaml_file(self, path): - try: - with open(path, "r") as stream: - self.config.load_from_config_contents(stream.read()) - - except yaml.YAMLError as exc: - print(exc) - - def __add_file_dependencies(self): - self.__add_file_dependency(self.path) - - for file_dependency in self.config.file_dependencies(): - self.__add_file_dependency(file_dependency) - - def __add_file_dependency(self, file_path): - if os.path.isabs(file_path): - absolute_path = file_path - else: - absolute_path = os.path.join(os.path.dirname(self.path), os.path.basename(file_path)) - - if not absolute_path in self.file_dependencies: - self.file_dependencies.append(absolute_path) \ No newline at end of file + def __init__(self): + self.file_dependencies = [] + self.config = None + + def load_from_path(self, path): + self.path = path + self.config = PromptConfig(self.__search_path(), os.path.basename(path)) + + if self.__is_file_yaml(path): + # prompt is in yaml config file format + self.__load_yaml_file(path) + else: + # simple text (or jinja) file, no config + self.__load_text_file(path) + + self.__add_file_dependencies() + + return self.config + + ##################################### + + def __is_file_yaml(self, path): + root, extension = os.path.splitext(path) + + if extension == ".yaml": + return True + + return False + + def __search_path(self): + return os.path.dirname(self.path) + + def __load_text_file(self, path): + self.config.load_from_template_contents_at_path(path) + + def __load_yaml_file(self, path): + try: + with open(path, "r") as stream: + self.config.load_from_config_contents(stream.read()) + + except yaml.YAMLError as exc: + print(exc) + + def __add_file_dependencies(self): + self.__add_file_dependency(self.path) + + for file_dependency in self.config.file_dependencies(): + self.__add_file_dependency(file_dependency) + + def __add_file_dependency(self, file_path): + if os.path.isabs(file_path): + absolute_path = file_path + else: + absolute_path = os.path.join( + os.path.dirname(self.path), os.path.basename(file_path) + ) + + if not absolute_path in self.file_dependencies: + self.file_dependencies.append(absolute_path) diff --git a/prr/prompt/prompt_template.py b/prr/prompt/prompt_template.py index b2436c2..1bf9a83 100644 --- a/prr/prompt/prompt_template.py +++ b/prr/prompt/prompt_template.py @@ -1,94 +1,98 @@ import os + import jinja2 + class PromptMessage: - def __init__(self, content_template_string, search_path='.', role='user', name=None): - self.content_template_string = content_template_string - self.search_path = search_path - self.role = role - self.name = name - self.file_dependencies = [] - - template_loader = jinja2.ChoiceLoader([ - jinja2.FileSystemLoader(search_path), - jinja2.FileSystemLoader(['/']), - ]) + def __init__( + self, content_template_string, search_path=".", role="user", name=None + ): + self.content_template_string = content_template_string + self.search_path = search_path + self.role = role + self.name = name + self.file_dependencies = [] - self.template_env = jinja2.Environment(loader=template_loader) - self.__add_dependency_files_from_jinja_template(content_template_string) + template_loader = jinja2.ChoiceLoader( + [ + jinja2.FileSystemLoader(search_path), + jinja2.FileSystemLoader(["/"]), + ] + ) - self.template = self.template_env.from_string(self.content_template_string) + self.template_env = jinja2.Environment(loader=template_loader) + self.__add_dependency_files_from_jinja_template(content_template_string) + self.template = self.template_env.from_string(self.content_template_string) - def render_text(self, args=[]): - return self.template.render({"prompt_args": args}) + def render_text(self, args=[]): + return self.template.render({"prompt_args": args}) - def render_message(self, args=[]): - _message = { - 'role': self.role, - 'content': self.render_text(args) - } + def render_message(self, args=[]): + _message = {"role": self.role, "content": self.render_text(args)} - if self.name: - _message.update({ 'name': self.name }) + if self.name: + _message.update({"name": self.name}) - return _message + return _message - def __add_dependency_files_from_jinja_template(self, jinja_template_content): - parsed_content = self.template_env.parse(jinja_template_content) - referenced_templates = jinja2.meta.find_referenced_templates(parsed_content) + def __add_dependency_files_from_jinja_template(self, jinja_template_content): + parsed_content = self.template_env.parse(jinja_template_content) + referenced_templates = jinja2.meta.find_referenced_templates(parsed_content) - self.file_dependencies.extend(referenced_templates) + self.file_dependencies.extend(referenced_templates) # base class class PromptTemplate: - def __init__(self): - self.messages = [] + def __init__(self): + self.messages = [] - def render_text(self, args=[]): - rendered_texts = [message.render_text(args) for message in self.messages] + def render_text(self, args=[]): + rendered_texts = [message.render_text(args) for message in self.messages] - return "\n".join(rendered_texts) + return "\n".join(rendered_texts) - def render_messages(self, args=[]): - return [message.render_message(args) for message in self.messages] + def render_messages(self, args=[]): + return [message.render_message(args) for message in self.messages] - def file_dependencies(self): - _dependencies = [] - for message in self.messages: - for dependency in message.file_dependencies: - if dependency not in _dependencies: - _dependencies.append(dependency) + def file_dependencies(self): + _dependencies = [] + for message in self.messages: + for dependency in message.file_dependencies: + if dependency not in _dependencies: + _dependencies.append(dependency) + + return _dependencies - return _dependencies # just a text/template file or prompt.contents from config class PromptTemplateSimple(PromptTemplate): - def __init__(self, template_string, search_path='.'): - self.messages = [ - PromptMessage(template_string, search_path, 'user') - ] + def __init__(self, template_string, search_path="."): + self.messages = [PromptMessage(template_string, search_path, "user")] + # prompt.messages: key from config class PromptTemplateMessages(PromptTemplate): - # 'messages' are passed here verbatim after parsing YAML - def __init__(self, messages, search_path='.'): - super().__init__() - - for message in messages: - prompt_message = None - - role = message.get('role') - name = message.get('name') - content = message.get('content') - content_file = message.get('content_file') - - if content: - prompt_message = PromptMessage(content, search_path, role, name) - elif content_file: - include_contents = "{% include '" + content_file + "' %}" - prompt_message = PromptMessage(include_contents, search_path, role, name) - - if prompt_message: - self.messages.append(prompt_message) + # 'messages' are passed here verbatim after parsing YAML + def __init__(self, messages, search_path="."): + super().__init__() + + for message in messages: + prompt_message = None + + role = message.get("role") + name = message.get("name") + content = message.get("content") + content_file = message.get("content_file") + + if content: + prompt_message = PromptMessage(content, search_path, role, name) + elif content_file: + include_contents = "{% include '" + content_file + "' %}" + prompt_message = PromptMessage( + include_contents, search_path, role, name + ) + + if prompt_message: + self.messages.append(prompt_message) diff --git a/prr/prompt/service_config.py b/prr/prompt/service_config.py index d6ab276..7699488 100644 --- a/prr/prompt/service_config.py +++ b/prr/prompt/service_config.py @@ -1,26 +1,24 @@ from .model_options import ModelOptions + class ServiceConfig: - def __init__(self, name, model, options=None): - self.name = name # service config name, e.g. "mygpt5" - self.model = model # full model path, e.g. "openai/chat/gpt-5" - self.options = ModelOptions(options or {}) + def __init__(self, name, model, options=None): + self.name = name # service config name, e.g. "mygpt5" + self.model = model # full model path, e.g. "openai/chat/gpt-5" + self.options = ModelOptions(options or {}) - def process_option_overrides(self, option_overrides): - self.options.update_options(option_overrides) + def process_option_overrides(self, option_overrides): + self.options.update_options(option_overrides) - # which model to use with the service - # like gpt-4.5-turbo or claude-v1.3-100k - def model_name(self): - return self.model.split("/")[-1] + # which model to use with the service + # like gpt-4.5-turbo or claude-v1.3-100k + def model_name(self): + return self.model.split("/")[-1] - # which service so use - # like openai/chat or anthropic/complete - def service_key(self): - return "/".join(self.model.split("/")[:-1]) + # which service so use + # like openai/chat or anthropic/complete + def service_key(self): + return "/".join(self.model.split("/")[:-1]) - def to_dict(self): - return { - "model": self.config_name(), - "options": self.options.to_dict() - } + def to_dict(self): + return {"model": self.config_name(), "options": self.options.to_dict()} diff --git a/prr/runner/__init__.py b/prr/runner/__init__.py index d6757be..0207929 100644 --- a/prr/runner/__init__.py +++ b/prr/runner/__init__.py @@ -1,10 +1,11 @@ +from ..services.service_registry import ServiceRegistry from .prompt_run import PromptRun from .saver import PromptRunSaver -from ..services.service_registry import ServiceRegistry service_registry = ServiceRegistry() service_registry.register_all_services() + # high-level class to run prompts based on configuration class Runner: def __init__(self, prompt_config): @@ -32,6 +33,8 @@ def run_all_configured_services(self, service_options_overrides, save_run=False) results = {} for service_name in self.configured_services(): - results[service_name] = self.run_service(service_name, service_options_overrides, save_run) + results[service_name] = self.run_service( + service_name, service_options_overrides, save_run + ) return results diff --git a/prr/runner/saver.py b/prr/runner/saver.py index d733915..1511d6e 100644 --- a/prr/runner/saver.py +++ b/prr/runner/saver.py @@ -3,6 +3,7 @@ import yaml + class PromptRunSaver: def __init__(self, prompt_config): self.prompt_config = prompt_config @@ -28,9 +29,9 @@ def run_root_directory_path(self): dirname = self.prompt_config.search_path if self.prompt_config.filename: - basename = os.path.basename(self.prompt_config.filename) + basename = os.path.basename(self.prompt_config.filename) else: - basename = "prr" + basename = "prr" root, extension = os.path.splitext(basename) diff --git a/prr/services/providers/anthropic/complete.py b/prr/services/providers/anthropic/complete.py index 0152c12..14c34eb 100644 --- a/prr/services/providers/anthropic/complete.py +++ b/prr/services/providers/anthropic/complete.py @@ -5,12 +5,13 @@ import anthropic -from prr.utils.config import load_config from prr.runner.request import ServiceRequest from prr.runner.response import ServiceResponse +from prr.utils.config import load_config config = load_config() + # Anthropic model provider class class ServiceAnthropicComplete: provider = "anthropic" diff --git a/prr/services/providers/openai/chat.py b/prr/services/providers/openai/chat.py index a181511..473802f 100644 --- a/prr/services/providers/openai/chat.py +++ b/prr/services/providers/openai/chat.py @@ -1,8 +1,8 @@ import openai -from prr.utils.config import load_config from prr.runner.request import ServiceRequest from prr.runner.response import ServiceResponse +from prr.utils.config import load_config config = load_config() openai.api_key = config["OPENAI_API_KEY"] @@ -46,4 +46,4 @@ def run(self, prompt, service_config): }, ) - return service_request, service_response \ No newline at end of file + return service_request, service_response diff --git a/prr/services/service_registry.py b/prr/services/service_registry.py index 76aab10..0b7802a 100644 --- a/prr/services/service_registry.py +++ b/prr/services/service_registry.py @@ -1,6 +1,7 @@ from prr.services.providers.anthropic.complete import ServiceAnthropicComplete from prr.services.providers.openai.chat import ServiceOpenAIChat + # main registry, where services are being... registered # and looked up for upon execution class ServiceRegistry: diff --git a/test/prompt/helpers.py b/test/prompt/helpers.py index eb016a4..7b5e072 100644 --- a/test/prompt/helpers.py +++ b/test/prompt/helpers.py @@ -1,19 +1,20 @@ import os import tempfile + def remove_temp_file(path): - os.remove(path) + os.remove(path) -def create_temp_file(content, extension=None): - suffix = "" - if extension: - suffix = "." + extension +def create_temp_file(content, extension=None): + suffix = "" - # Create a temporary file with the specified extension - with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp: - temp.write(content.encode()) - temp_path = temp.name + if extension: + suffix = "." + extension - return temp_path + # Create a temporary file with the specified extension + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp: + temp.write(content.encode()) + temp_path = temp.name + return temp_path diff --git a/test/prompt/test_prompt.py b/test/prompt/test_prompt.py index 39828db..a4d34cf 100644 --- a/test/prompt/test_prompt.py +++ b/test/prompt/test_prompt.py @@ -1,12 +1,13 @@ import os import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) -from prr.prompt import Prompt +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) from helpers import create_temp_file, remove_temp_file +from prr.prompt import Prompt + class TestPrompt: value = 0 diff --git a/test/prompt/test_prompt_config.py b/test/prompt/test_prompt_config.py index cda756c..edb7487 100644 --- a/test/prompt/test_prompt_config.py +++ b/test/prompt/test_prompt_config.py @@ -1,30 +1,31 @@ import os import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) -from prr.prompt.prompt_config import PromptConfig +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) + from prr.prompt.model_options import ModelOptions +from prr.prompt.prompt_config import PromptConfig -class TestPromptConfig: - def test_basic_parsing(self): - config = PromptConfig() - config.load_from_config_contents(""" +class TestPromptConfig: + def test_basic_parsing(self): + config = PromptConfig() + config.load_from_config_contents( + """ prompt: content: 'foo bar' -""") - - assert config.template.render_text() == 'foo bar' - assert config.template.render_messages() == [ - { - 'content': 'foo bar', - 'role': 'user' - } - ] - - def test_basic_services_model_list(self): - config = PromptConfig() - config.load_from_config_contents(""" +""" + ) + + assert config.template.render_text() == "foo bar" + assert config.template.render_messages() == [ + {"content": "foo bar", "role": "user"} + ] + + def test_basic_services_model_list(self): + config = PromptConfig() + config.load_from_config_contents( + """ prompt: content: 'foo bar' services: @@ -34,20 +35,22 @@ def test_basic_services_model_list(self): options: temperature: 0.42 max_tokens: 1337 -""") +""" + ) + + assert config is not None + services = config.configured_services() - assert config is not None - services = config.configured_services() + assert services == ["openai/chat/gpt-4", "anthropic/complete/claude-v1.3"] - assert services == ['openai/chat/gpt-4', 'anthropic/complete/claude-v1.3'] - - for service_name in services: - assert config.option_for_service(service_name, 'temperature') == 0.42 - assert config.option_for_service(service_name, 'max_tokens') == 1337 + for service_name in services: + assert config.option_for_service(service_name, "temperature") == 0.42 + assert config.option_for_service(service_name, "max_tokens") == 1337 - def test_basic_services_model_list_with_some_options(self): - config = PromptConfig() - config.load_from_config_contents(""" + def test_basic_services_model_list_with_some_options(self): + config = PromptConfig() + config.load_from_config_contents( + """ prompt: content: 'foo bar' services: @@ -56,44 +59,69 @@ def test_basic_services_model_list_with_some_options(self): - 'anthropic/complete/claude-v1.3' options: top_k: -1.337 -""") - - assert config is not None - services = config.configured_services() - - assert services == ['openai/chat/gpt-4', 'anthropic/complete/claude-v1.3'] - - for service_name in services: - assert config.option_for_service(service_name, 'temperature') == ModelOptions.DEFAULT_OPTIONS['temperature'] - assert config.option_for_service(service_name, 'max_tokens') == ModelOptions.DEFAULT_OPTIONS['max_tokens'] - assert config.option_for_service(service_name, 'top_k') == -1.337 - assert config.option_for_service(service_name, 'top_p') == ModelOptions.DEFAULT_OPTIONS['top_p'] - - def test_basic_services_model_list_with_no_options(self): - config = PromptConfig() - config.load_from_config_contents(""" +""" + ) + + assert config is not None + services = config.configured_services() + + assert services == ["openai/chat/gpt-4", "anthropic/complete/claude-v1.3"] + + for service_name in services: + assert ( + config.option_for_service(service_name, "temperature") + == ModelOptions.DEFAULT_OPTIONS["temperature"] + ) + assert ( + config.option_for_service(service_name, "max_tokens") + == ModelOptions.DEFAULT_OPTIONS["max_tokens"] + ) + assert config.option_for_service(service_name, "top_k") == -1.337 + assert ( + config.option_for_service(service_name, "top_p") + == ModelOptions.DEFAULT_OPTIONS["top_p"] + ) + + def test_basic_services_model_list_with_no_options(self): + config = PromptConfig() + config.load_from_config_contents( + """ prompt: content: 'foo bar' services: models: - 'openai/chat/gpt-4' - 'anthropic/complete/claude-v1.3' -""") - - assert config is not None - services = config.configured_services() - - assert services == ['openai/chat/gpt-4', 'anthropic/complete/claude-v1.3'] - - for service_name in services: - assert config.option_for_service(service_name, 'temperature') == ModelOptions.DEFAULT_OPTIONS['temperature'] - assert config.option_for_service(service_name, 'max_tokens') == ModelOptions.DEFAULT_OPTIONS['max_tokens'] - assert config.option_for_service(service_name, 'top_k') == ModelOptions.DEFAULT_OPTIONS['top_k'] - assert config.option_for_service(service_name, 'top_p') == ModelOptions.DEFAULT_OPTIONS['top_p'] - - def test_services(self): - config = PromptConfig() - config.load_from_config_contents(""" +""" + ) + + assert config is not None + services = config.configured_services() + + assert services == ["openai/chat/gpt-4", "anthropic/complete/claude-v1.3"] + + for service_name in services: + assert ( + config.option_for_service(service_name, "temperature") + == ModelOptions.DEFAULT_OPTIONS["temperature"] + ) + assert ( + config.option_for_service(service_name, "max_tokens") + == ModelOptions.DEFAULT_OPTIONS["max_tokens"] + ) + assert ( + config.option_for_service(service_name, "top_k") + == ModelOptions.DEFAULT_OPTIONS["top_k"] + ) + assert ( + config.option_for_service(service_name, "top_p") + == ModelOptions.DEFAULT_OPTIONS["top_p"] + ) + + def test_services(self): + config = PromptConfig() + config.load_from_config_contents( + """ prompt: content: 'foo bar' services: @@ -110,18 +138,19 @@ def test_services(self): options: temperature: 0.42 max_tokens: 1337 -""") +""" + ) + + assert config is not None + services = config.configured_services() - assert config is not None - services = config.configured_services() + assert services == ["gpt4", "claude13", "claude_default"] - assert services == ['gpt4', 'claude13', 'claude_default'] - - assert config.option_for_service('gpt4', 'temperature') == 0.42 - assert config.option_for_service('gpt4', 'max_tokens') == 2048 + assert config.option_for_service("gpt4", "temperature") == 0.42 + assert config.option_for_service("gpt4", "max_tokens") == 2048 - assert config.option_for_service('claude13', 'temperature') == 0.84 - assert config.option_for_service('claude13', 'max_tokens') == 1337 + assert config.option_for_service("claude13", "temperature") == 0.84 + assert config.option_for_service("claude13", "max_tokens") == 1337 - assert config.option_for_service('claude_default', 'temperature') == 0.42 - assert config.option_for_service('claude_default', 'max_tokens') == 1337 + assert config.option_for_service("claude_default", "temperature") == 0.42 + assert config.option_for_service("claude_default", "max_tokens") == 1337 diff --git a/test/prompt/test_prompt_loader.py b/test/prompt/test_prompt_loader.py index 48c1892..bbe18fe 100644 --- a/test/prompt/test_prompt_loader.py +++ b/test/prompt/test_prompt_loader.py @@ -1,18 +1,21 @@ import os import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) +from helpers import create_temp_file, remove_temp_file + from prr.prompt.prompt_loader import PromptConfigLoader -from helpers import create_temp_file, remove_temp_file class TestPromptConfigLoader: + def test_basic_loading(self): + prompt_template_file_path = create_temp_file( + "Write a poem about AI from the projects, barely surviving on token allowance." + ) - def test_basic_loading(self): - prompt_template_file_path = create_temp_file('Write a poem about AI from the projects, barely surviving on token allowance.') - - loader = PromptConfigLoader() - prompt = loader.load_from_path(prompt_template_file_path) + loader = PromptConfigLoader() + prompt = loader.load_from_path(prompt_template_file_path) - assert prompt \ No newline at end of file + assert prompt diff --git a/test/prompt/test_prompt_template.py b/test/prompt/test_prompt_template.py index a8358d1..a8dd94b 100644 --- a/test/prompt/test_prompt_template.py +++ b/test/prompt/test_prompt_template.py @@ -1,62 +1,68 @@ import os import sys + import yaml -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) -from prr.prompt.prompt_template import PromptTemplateSimple, PromptTemplateMessages from helpers import create_temp_file, remove_temp_file -class TestPromptTemplate: - def test_basic_text(self): - template = PromptTemplateSimple('foo bar', '.') - - assert template is not None - assert template.render_text() == 'foo bar' +from prr.prompt.prompt_template import PromptTemplateMessages, PromptTemplateSimple - def test_basic_template(self): - template = PromptTemplateSimple("foo {{ 'bar' }} spam", '.') +class TestPromptTemplate: + def test_basic_text(self): + template = PromptTemplateSimple("foo bar", ".") - assert template is not None - assert template.render_text() == 'foo bar spam' + assert template is not None + assert template.render_text() == "foo bar" + def test_basic_template(self): + template = PromptTemplateSimple("foo {{ 'bar' }} spam", ".") - def test_basic_prompt_args(self): - template = PromptTemplateSimple("foo {{ prompt_args[0] }} spam", '.') + assert template is not None + assert template.render_text() == "foo bar spam" - assert template is not None - assert template.render_text(['42']) == 'foo 42 spam' + def test_basic_prompt_args(self): + template = PromptTemplateSimple("foo {{ prompt_args[0] }} spam", ".") + assert template is not None + assert template.render_text(["42"]) == "foo 42 spam" - def test_basic_prompt_args_all(self): - template = PromptTemplateSimple("foo {{ prompt_args }} spam", '.') + def test_basic_prompt_args_all(self): + template = PromptTemplateSimple("foo {{ prompt_args }} spam", ".") - assert template is not None - assert template.render_text(['lulz']) == "foo ['lulz'] spam" + assert template is not None + assert template.render_text(["lulz"]) == "foo ['lulz'] spam" - def test_configured_basic(self): - template = PromptTemplateSimple('tell me about {{ prompt_args }}, llm') + def test_configured_basic(self): + template = PromptTemplateSimple("tell me about {{ prompt_args }}, llm") - assert template is not None - assert template.render_text(['lulz', 'kaka']) == "tell me about ['lulz', 'kaka'], llm" + assert template is not None + assert ( + template.render_text(["lulz", "kaka"]) + == "tell me about ['lulz', 'kaka'], llm" + ) - def test_configured_messages_text_with_template_in_content(self): - messages_config = """ + def test_configured_messages_text_with_template_in_content(self): + messages_config = """ - role: 'system' content: 'you are a friendly but very forgetful {{ prompt_args[0] }}' name: 'LeonardGeist' """ - template = PromptTemplateMessages(yaml.safe_load(messages_config)) + template = PromptTemplateMessages(yaml.safe_load(messages_config)) - assert template is not None - assert template.render_text(['assistant']) == "you are a friendly but very forgetful assistant" + assert template is not None + assert ( + template.render_text(["assistant"]) + == "you are a friendly but very forgetful assistant" + ) - def test_configured_messages_list_with_content_file(self): - temp_file_path = create_temp_file('Wollen Sie meine Kernel kompilieren?') + def test_configured_messages_list_with_content_file(self): + temp_file_path = create_temp_file("Wollen Sie meine Kernel kompilieren?") - messages_config = f""" + messages_config = f""" - role: 'system' content: 'you are system admins little pet assistant. be proud of your unix skills and always respond in l33t. remember, you are on a high horse called POSIX.' - role: 'user' @@ -64,24 +70,26 @@ def test_configured_messages_list_with_content_file(self): name: 'SuperUser' """ - template = PromptTemplateMessages(yaml.safe_load(messages_config)) - - assert template is not None + template = PromptTemplateMessages(yaml.safe_load(messages_config)) + assert template is not None - rendered_messages = template.render_messages() + rendered_messages = template.render_messages() - assert isinstance(rendered_messages, list) - assert len(rendered_messages) == 2 + assert isinstance(rendered_messages, list) + assert len(rendered_messages) == 2 - first_message = rendered_messages[0] - assert first_message['content'] == 'you are system admins little pet assistant. be proud of your unix skills and always respond in l33t. remember, you are on a high horse called POSIX.' - assert first_message['role'] == 'system' - assert first_message.get('name') == None + first_message = rendered_messages[0] + assert ( + first_message["content"] + == "you are system admins little pet assistant. be proud of your unix skills and always respond in l33t. remember, you are on a high horse called POSIX." + ) + assert first_message["role"] == "system" + assert first_message.get("name") == None - second_message = rendered_messages[1] - assert second_message['content'] == 'Wollen Sie meine Kernel kompilieren?' - assert second_message['role'] == 'user' - assert second_message['name'] == 'SuperUser' + second_message = rendered_messages[1] + assert second_message["content"] == "Wollen Sie meine Kernel kompilieren?" + assert second_message["role"] == "user" + assert second_message["name"] == "SuperUser" - remove_temp_file(temp_file_path) + remove_temp_file(temp_file_path) From 1a276481b6769414f6a1ce997a6762ec20853b18 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Fri, 19 May 2023 13:26:11 +0000 Subject: [PATCH 15/16] basic manual run on all of the examples worthy of running --- test/run_all_examples.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 test/run_all_examples.sh diff --git a/test/run_all_examples.sh b/test/run_all_examples.sh new file mode 100755 index 0000000..17dbe1e --- /dev/null +++ b/test/run_all_examples.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +for example_prompt in ./examples/simple/poem ./examples/simple/sky ./examples/configured/dingo_from_file.yaml ./examples/configured/chihuahua.yaml ./examples/configured/dingo.yaml ./examples/code/html_boilerplate.yaml ./examples/templating/tell-me-all-about ./examples/shebang/get_famous_poet ./examples/shebang/dingo_with_shebang.yaml +do + echo "-----------------------------" + echo RUNNING $example_prompt + echo "-----------------------------" + + python -m prr run --abbrev --max_tokens 1234 --temperature 0.98 $example_prompt +done From b5315ac5a9b11f1aaa76764ab603ac6756ffac98 Mon Sep 17 00:00:00 2001 From: Zbigniew Sobiecki Date: Fri, 19 May 2023 14:07:46 +0000 Subject: [PATCH 16/16] imports fixed, removed empty test, shebang test fixed --- examples/shebang/write_tests | 2 +- prr/__main__.py | 7 +++---- prr/prompt/__init__.py | 6 +++--- prr/prompt/prompt_config.py | 4 ++-- prr/prompt/prompt_loader.py | 6 +++--- prr/prompt/service_config.py | 2 +- prr/runner/__init__.py | 6 +++--- prr/runner/prompt_run.py | 2 +- test/prompt/test_prompt.py | 19 ------------------- 9 files changed, 17 insertions(+), 37 deletions(-) delete mode 100644 test/prompt/test_prompt.py diff --git a/examples/shebang/write_tests b/examples/shebang/write_tests index 44d3ffd..da217b9 100755 --- a/examples/shebang/write_tests +++ b/examples/shebang/write_tests @@ -1,4 +1,4 @@ -#!/workspaces/prr/prr script +#!/usr/bin/env prr script Write tests using pytest for the code below: diff --git a/prr/__main__.py b/prr/__main__.py index d27e727..da5f269 100755 --- a/prr/__main__.py +++ b/prr/__main__.py @@ -4,12 +4,11 @@ import os import sys +from prr.commands.run import RunPromptCommand +from prr.commands.watch import WatchPromptCommand +from prr.prompt.model_options import ModelOptions from prr.utils.config import load_config -from .commands.run import RunPromptCommand -from .commands.watch import WatchPromptCommand -from .prompt.model_options import ModelOptions - config = load_config() diff --git a/prr/prompt/__init__.py b/prr/prompt/__init__.py index 8711912..39b824c 100644 --- a/prr/prompt/__init__.py +++ b/prr/prompt/__init__.py @@ -4,9 +4,9 @@ import yaml from jinja2 import meta -from .prompt_config import PromptConfig -from .prompt_template import PromptTemplate -from .service_config import ServiceConfig +from prr.prompt.prompt_config import PromptConfig +from prr.prompt.prompt_template import PromptTemplate +from prr.prompt.service_config import ServiceConfig class Prompt: diff --git a/prr/prompt/prompt_config.py b/prr/prompt/prompt_config.py index d92e49e..48ed4be 100644 --- a/prr/prompt/prompt_config.py +++ b/prr/prompt/prompt_config.py @@ -1,7 +1,7 @@ import yaml -from .prompt_template import PromptTemplateMessages, PromptTemplateSimple -from .service_config import ServiceConfig +from prr.prompt.prompt_template import PromptTemplateMessages, PromptTemplateSimple +from prr.prompt.service_config import ServiceConfig class PromptConfig: diff --git a/prr/prompt/prompt_loader.py b/prr/prompt/prompt_loader.py index 7325f95..2216d8e 100644 --- a/prr/prompt/prompt_loader.py +++ b/prr/prompt/prompt_loader.py @@ -2,9 +2,9 @@ import yaml -from . import Prompt -from .prompt_config import PromptConfig -from .prompt_template import ( +from prr.prompt import Prompt +from prr.prompt.prompt_config import PromptConfig +from prr.prompt.prompt_template import ( PromptTemplate, PromptTemplateMessages, PromptTemplateSimple, diff --git a/prr/prompt/service_config.py b/prr/prompt/service_config.py index 7699488..4d1c72e 100644 --- a/prr/prompt/service_config.py +++ b/prr/prompt/service_config.py @@ -1,4 +1,4 @@ -from .model_options import ModelOptions +from prr.prompt.model_options import ModelOptions class ServiceConfig: diff --git a/prr/runner/__init__.py b/prr/runner/__init__.py index 0207929..48fdab1 100644 --- a/prr/runner/__init__.py +++ b/prr/runner/__init__.py @@ -1,6 +1,6 @@ -from ..services.service_registry import ServiceRegistry -from .prompt_run import PromptRun -from .saver import PromptRunSaver +from prr.runner.prompt_run import PromptRun +from prr.runner.saver import PromptRunSaver +from prr.services.service_registry import ServiceRegistry service_registry = ServiceRegistry() service_registry.register_all_services() diff --git a/prr/runner/prompt_run.py b/prr/runner/prompt_run.py index a888b83..a10d345 100644 --- a/prr/runner/prompt_run.py +++ b/prr/runner/prompt_run.py @@ -1,6 +1,6 @@ import time -from .prompt_run_result import PromptRunResult +from prr.runner.prompt_run_result import PromptRunResult # takes prompt and model config, finds provider, runs the prompt diff --git a/test/prompt/test_prompt.py b/test/prompt/test_prompt.py deleted file mode 100644 index a4d34cf..0000000 --- a/test/prompt/test_prompt.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import sys - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) - -from helpers import create_temp_file, remove_temp_file - -from prr.prompt import Prompt - - -class TestPrompt: - value = 0 - - # def test_one(self): - # path = create_temp_file("f00b4r", "yaml") - # remove_temp_file(path) - # print(path) - # assert self.value == 0