Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added integration tests for meta #274

Merged
merged 3 commits into from
Nov 8, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,44 @@ def test_tool_calling_with_no_arguments(self, model: BaseChatModel) -> None:
pass


class TestBedrockMetaStandard(ChatModelIntegrationTests):
@property
def chat_model_class(self) -> Type[BaseChatModel]:
return ChatBedrockConverse

@property
def chat_model_params(self) -> dict:
return {"model": "us.meta.llama3-2-90b-instruct-v1:0"}

@property
def standard_chat_model_params(self) -> dict:
return {"temperature": 0.1, "max_tokens": 100, "stop": []}

@pytest.mark.xfail(reason="Meta models don't support tool_choice.")
def test_structured_few_shot_examples(self, model: BaseChatModel) -> None:
pass

# TODO: This needs investigation, if this is a bug with Bedrock or Llama models,
# but this test consistently seem to return single quoted input values {input: '3'}
# instead of {input: 3} failing the test. Upon checking with tools with non-numeric
# inputs, tool calling seems to work as expected with Bedrock and Llama models.
@pytest.mark.xfail(
reason="Bedrock Meta models tend to return string values for integer inputs ."
)
def test_tool_calling(self, model: BaseChatModel) -> None:
super().test_tool_calling(model)

@pytest.mark.xfail(reason="Meta models don't support tool_choice.")
def test_tool_calling_with_no_arguments(self, model: BaseChatModel) -> None:
pass

@pytest.mark.xfail(
reason="Human messages following AI messages not supported by Bedrock."
)
def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None:
super().test_tool_message_histories_list_content(model)


def test_structured_output_snake_case() -> None:
model = ChatBedrockConverse(
model="anthropic.claude-3-sonnet-20240229-v1:0", temperature=0
Expand Down
Loading