-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature: added method run multi prompts
Feature method run multi prompts
- Loading branch information
Showing
6 changed files
with
120 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
[ | ||
{ | ||
"SYS": "You are a technical documentation writer. You always write clear, concise, and accurate documentation for\nscientific experiments. Your documentation focuses on the experiment's purpose, procedure, and results. Therefore,\ndetails about specific python functions, packages, or libraries are not necessary. Your readers are experimental\nscientists.", | ||
"INSTR": "Please generate high-level one or two paragraph documentation for the following experiment." | ||
}, | ||
{ | ||
"SYS": "You are a technical documentation writer. You always write clear, concise, and accurate documentation\nfor scientific experiments. Your documentation focuses on the experiment's procedure. Therefore, details about specific\npython functions, packages, or libraries are NOT necessary. Your readers are experimental scientists.\nFor writing your descriptions, follow these instructions:\n- DO NOT write greetings or preambles\n- Use the Variable 'name' attribute and not the python variable names\n- Use LaTeX for math expressions\n- DO NOT include code or code-like syntax and do not use python function or class names\n- Write in paragraph style, NOT bullet points", | ||
"INSTR": "Generate a one line description of the dependent and independent variables used in the following\npython code: " | ||
}, | ||
{ | ||
"SYS": "You are a research scientist. You always write clear, concise, and accurate documentation\nfor scientific experiments from python code. Your documentation focuses on the experiment's procedure. Therefore, details about specific\npython functions, packages, or libraries are NOT necessary. Your readers are experimental scientists.\nFor writing your descriptions, follow these instructions:\n- DO NOT write greetings or preambles\n- Use the Variable 'name' attribute and not the python variable names\n- Use LaTeX for math expressions\n- DO NOT include code or code-like syntax and do not use python function or class names\n- Write in paragraph style, NOT bullet points", | ||
"INSTR": "Generate a three line description of the dependent and independent variables used in the following\npython code: " | ||
} | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
from dataclasses import dataclass | ||
from typing import List, Optional | ||
|
||
|
||
@dataclass | ||
class EvalResult: | ||
"""Class for storing LLM evaluation results""" | ||
|
||
prediction: List[str] | ||
prompt: str | ||
bleu_score: Optional[float] = None | ||
meteor_score: Optional[float] = None |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
import json | ||
from typing import Any, Dict, List, Tuple | ||
|
||
from autora.doc.runtime.prompts import PromptBuilder | ||
|
||
|
||
def load_file(json_file_path: str) -> List[Dict[str, Any]]: | ||
# Read and parse the JSON file | ||
with open(json_file_path, "r") as file: | ||
data: List[Dict[str, Any]] = json.load(file) | ||
return data | ||
|
||
|
||
def get_prompts_from_file(prompts_file: str) -> List[str]: | ||
prompts_data = load_file(prompts_file) | ||
prompts_list = [PromptBuilder(p["SYS"], p["INSTR"]).build() for p in prompts_data] | ||
return prompts_list |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
from pathlib import Path | ||
|
||
from autora.doc.util import get_prompts_from_file, load_file | ||
|
||
|
||
def test_load_file() -> None: | ||
prompts_file_path = Path(__file__).parent.joinpath("../data/autora/prompts/all_prompt.json").resolve() | ||
data = load_file(str(prompts_file_path)) | ||
assert type(data) == list | ||
|
||
|
||
def test_get_prompts_from_file() -> None: | ||
prompts_file_path = Path(__file__).parent.joinpath("../data/autora/prompts/all_prompt.json").resolve() | ||
prompts_list = get_prompts_from_file(str(prompts_file_path)) | ||
|
||
assert len(prompts_list) == 3, "Expected 3 outputs" | ||
for prompt in prompts_list: | ||
assert type(prompt) == str |