-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Co-authored-by: Matt Zhou <[email protected]>
- Loading branch information
Showing
8 changed files
with
133 additions
and
41 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| name: Run CLI tests | ||
|
|
||
| env: | ||
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | ||
|
|
||
| on: | ||
| push: | ||
| branches: [ main ] | ||
| pull_request: | ||
| branches: [ main ] | ||
|
|
||
| jobs: | ||
| test: | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 15 | ||
|
|
||
| services: | ||
| qdrant: | ||
| image: qdrant/qdrant | ||
| ports: | ||
| - 6333:6333 | ||
|
|
||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
|
|
||
| - name: Build and run container | ||
| run: bash db/run_postgres.sh | ||
|
|
||
| - name: "Setup Python, Poetry and Dependencies" | ||
| uses: packetcoders/action-setup-cache-python-poetry@main | ||
| with: | ||
| python-version: "3.12" | ||
| poetry-version: "1.8.2" | ||
| install-args: "-E dev -E postgres -E tests" | ||
|
|
||
| - name: Test `letta run` up until first message | ||
| env: | ||
| LETTA_PG_PORT: 8888 | ||
| LETTA_PG_USER: letta | ||
| LETTA_PG_PASSWORD: letta | ||
| LETTA_PG_DB: letta | ||
| LETTA_PG_HOST: localhost | ||
| LETTA_SERVER_PASS: test_server_token | ||
| run: | | ||
| poetry run pytest -s -vv tests/test_cli.py::test_letta_run_create_new_agent |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,44 +1,73 @@ | ||
| import subprocess | ||
| import os | ||
| import shutil | ||
| import sys | ||
|
|
||
| subprocess.check_call([sys.executable, "-m", "pip", "install", "pexpect"]) | ||
| from prettytable.colortable import ColorTable | ||
| import pexpect | ||
| import pytest | ||
|
|
||
| from letta.cli.cli_config import ListChoice, add, delete | ||
| from letta.cli.cli_config import list as list_command | ||
| from letta.local_llm.constants import ( | ||
| ASSISTANT_MESSAGE_CLI_SYMBOL, | ||
| INNER_THOUGHTS_CLI_SYMBOL, | ||
| ) | ||
|
|
||
| # def test_configure_letta(): | ||
| # configure_letta() | ||
| original_letta_path = os.path.expanduser("~/.letta") | ||
| backup_letta_path = os.path.expanduser("~/.letta_backup") | ||
|
|
||
| options = [ListChoice.agents, ListChoice.sources, ListChoice.humans, ListChoice.personas] | ||
|
|
||
| @pytest.fixture | ||
| def swap_letta_config(): | ||
| if os.path.exists(backup_letta_path): | ||
| print("\nDelete the backup ~/.letta directory\n") | ||
| shutil.rmtree(backup_letta_path) | ||
|
|
||
| def test_cli_list(): | ||
| for option in options: | ||
| output = list_command(arg=option) | ||
| # check if is a list | ||
| assert isinstance(output, ColorTable) | ||
| if os.path.exists(original_letta_path): | ||
| print("\nBackup the original ~/.letta directory\n") | ||
| shutil.move(original_letta_path, backup_letta_path) | ||
|
|
||
| try: | ||
| # Run the test | ||
| yield | ||
| finally: | ||
| # Ensure this runs no matter what | ||
| print("\nClean up ~/.letta and restore the original directory\n") | ||
| if os.path.exists(original_letta_path): | ||
| shutil.rmtree(original_letta_path) | ||
|
|
||
| def test_cli_config(): | ||
| if os.path.exists(backup_letta_path): | ||
| shutil.move(backup_letta_path, original_letta_path) | ||
|
|
||
| # test add | ||
| for option in ["human", "persona"]: | ||
|
|
||
| # create initial | ||
| add(option=option, name="test", text="test data") | ||
| def test_letta_run_create_new_agent(swap_letta_config): | ||
| child = pexpect.spawn("poetry run letta run", encoding="utf-8") | ||
| # Start the letta run command | ||
| child.logfile = sys.stdout | ||
| child.expect("Creating new agent", timeout=10) | ||
| # Optional: LLM model selection | ||
| try: | ||
| child.expect("Select LLM model:", timeout=10) | ||
| child.sendline("\033[B\033[B\033[B\033[B\033[B") | ||
| except (pexpect.TIMEOUT, pexpect.EOF): | ||
| print("[WARNING] LLM model selection step was skipped.") | ||
|
|
||
| ## update | ||
| # filename = "test.txt" | ||
| # open(filename, "w").write("test data new") | ||
| # child = pexpect.spawn(f"poetry run letta add --{str(option)} {filename} --name test --strip-ui") | ||
| # child.expect("Human test already exists. Overwrite?", timeout=TIMEOUT) | ||
| # child.sendline() | ||
| # child.expect(pexpect.EOF, timeout=TIMEOUT) # Wait for child to exit | ||
| # child.close() | ||
| # Optional: Embedding model selection | ||
| try: | ||
| child.expect("Select embedding model:", timeout=10) | ||
| child.sendline("text-embedding-ada-002") | ||
| except (pexpect.TIMEOUT, pexpect.EOF): | ||
| print("[WARNING] Embedding model selection step was skipped.") | ||
|
|
||
| for row in list_command(arg=ListChoice.humans if option == "human" else ListChoice.personas): | ||
| if row[0] == "test": | ||
| assert "test data" in row | ||
| # delete | ||
| delete(option=option, name="test") | ||
| child.expect("Created new agent", timeout=10) | ||
| child.sendline("") | ||
|
|
||
| # Get initial response | ||
| child.expect("Enter your message:", timeout=60) | ||
| # Capture the output up to this point | ||
| full_output = child.before | ||
| # Count occurrences of inner thoughts | ||
| cloud_emoji_count = full_output.count(INNER_THOUGHTS_CLI_SYMBOL) | ||
| assert cloud_emoji_count == 1, f"It appears that there are multiple instances of inner thought outputted." | ||
| # Count occurrences of assistant messages | ||
| robot = full_output.count(ASSISTANT_MESSAGE_CLI_SYMBOL) | ||
| assert robot == 1, f"It appears that there are multiple instances of assistant messages outputted." | ||
| # Make sure the user name was repeated back at least once | ||
| assert full_output.count("Chad") > 0, f"Chad was not mentioned...please manually inspect the outputs." |