Skip to content

Commit

Permalink
VertexAI: migrate test cases to use updated set of golden files (#8324)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tanzimfh
tanzimfh authored Jun 20, 2024
1 parent 576c1b1 commit ca41391
Show file tree
Hide file tree
Showing 17 changed files with 150 additions and 147 deletions.
10 changes: 3 additions & 7 deletions packages/vertexai/src/methods/count-tokens.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,7 @@ describe('countTokens()', () => {
restore();
});
it('total tokens', async () => {
const mockResponse = getMockResponse(
'count-tokens-success-total-tokens.json'
);
const mockResponse = getMockResponse('unary-success-total-tokens.json');
const makeRequestStub = stub(request, 'makeRequest').resolves(
mockResponse as Response
);
Expand All @@ -70,7 +68,7 @@ describe('countTokens()', () => {
});
it('total tokens no billable characters', async () => {
const mockResponse = getMockResponse(
'count-tokens-success-no-billable-characters.json'
'unary-success-no-billable-characters.json'
);
const makeRequestStub = stub(request, 'makeRequest').resolves(
mockResponse as Response
Expand All @@ -94,9 +92,7 @@ describe('countTokens()', () => {
);
});
it('model not found', async () => {
const mockResponse = getMockResponse(
'count-tokens-failure-model-not-found.json'
);
const mockResponse = getMockResponse('unary-failure-model-not-found.json');
const mockFetch = stub(globalThis, 'fetch').resolves({
ok: false,
status: 404,
Expand Down
10 changes: 6 additions & 4 deletions packages/vertexai/src/methods/generate-content.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ describe('generateContent()', () => {
'model',
fakeRequestParams
);
expect(result.response.text()).to.include('Helena');
expect(result.response.text()).to.include('Mountain View, California');
expect(makeRequestStub).to.be.calledWith(
'model',
Task.GENERATE_CONTENT,
Expand Down Expand Up @@ -112,10 +112,12 @@ describe('generateContent()', () => {
'model',
fakeRequestParams
);
expect(result.response.text()).to.include('Quantum mechanics is');
expect(result.response.text()).to.include(
'Some information cited from an external source'
);
expect(
result.response.candidates?.[0].citationMetadata?.citations.length
).to.equal(1);
).to.equal(3);
expect(makeRequestStub).to.be.calledWith(
'model',
Task.GENERATE_CONTENT,
Expand Down Expand Up @@ -186,7 +188,7 @@ describe('generateContent()', () => {
);
});
it('unknown enum - should ignore', async () => {
const mockResponse = getMockResponse('unary-unknown-enum.json');
const mockResponse = getMockResponse('unary-success-unknown-enum.json');
const makeRequestStub = stub(request, 'makeRequest').resolves(
mockResponse as Response
);
Expand Down
4 changes: 1 addition & 3 deletions packages/vertexai/src/models/generative-model.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,9 +264,7 @@ describe('GenerativeModel', () => {
});
it('calls countTokens', async () => {
const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model' });
const mockResponse = getMockResponse(
'count-tokens-success-total-tokens.json'
);
const mockResponse = getMockResponse('unary-success-total-tokens.json');
const makeRequestStub = stub(request, 'makeRequest').resolves(
mockResponse as Response
);
Expand Down
4 changes: 3 additions & 1 deletion packages/vertexai/src/requests/stream-reader.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,9 @@ describe('processStream', () => {
}
});
it('unknown enum - should ignore', async () => {
const fakeResponse = getMockResponseStreaming('streaming-unknown-enum.txt');
const fakeResponse = getMockResponseStreaming(
'streaming-success-unknown-enum.txt'
);
const result = processStream(fakeResponse as Response);
const aggregatedResponse = await result.response;
expect(aggregatedResponse.text()).to.include('Cats');
Expand Down
2 changes: 1 addition & 1 deletion packages/vertexai/test-utils/mock-responses/streaming-failure-finish-reason-safety.txt
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
data: {"candidates": [{"content": {"parts": [{"text": "No"}]},"finishReason": "SAFETY","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "HIGH"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}
data: {"candidates": [{"content": {"parts": [{"text": "<redacted>"}],"role": "model"},"finishReason": "SAFETY","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "HIGH"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}

6 changes: 3 additions & 3 deletions packages/vertexai/test-utils/mock-responses/streaming-success-basic-reply-long.txt
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
data: {"candidates": [{"content": {"parts": [{"text": "**Cats:**\n\n- **Physical Characteristics:**\n - Size: Cats come"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}
data: {"candidates": [{"content": {"parts": [{"text": "**Cats:**\n\n- **Physical Characteristics:**\n - Size: Cats come"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}

data: {"candidates": [{"content": {"parts": [{"text": " in a wide range of sizes, from small breeds like the Singapura to large breeds like the Maine Coon.\n - Fur: Cats have soft, furry coats"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}
data: {"candidates": [{"content": {"parts": [{"text": " in a wide range of sizes, from small breeds like the Singapura to large breeds like the Maine Coon.\n - Fur: Cats have soft, furry coats"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

data: {"candidates": [{"content": {"parts": [{"text": " that can vary in length and texture depending on the breed.\n - Eyes: Cats have large, expressive eyes that can be various colors, including green, blue, yellow, and hazel.\n - Ears: Cats have pointed, erect ears that are sensitive to sound.\n - Tail: Cats have long"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}
data: {"candidates": [{"content": {"parts": [{"text": " that can vary in length and texture depending on the breed.\n - Eyes: Cats have large, expressive eyes that can be various colors, including green, blue, yellow, and hazel.\n - Ears: Cats have pointed, erect ears that are sensitive to sound.\n - Tail: Cats have long"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

data: {"candidates": [{"content": {"parts": [{"text": ", flexible tails that they use for balance and communication.\n\n- **Behavior and Personality:**\n - Independent: Cats are often described as independent animals that enjoy spending time alone.\n - Affectionate: Despite their independent nature, cats can be very affectionate and form strong bonds with their owners.\n - Playful: Cats are naturally playful and enjoy engaging in activities such as chasing toys, climbing, and pouncing.\n - Curious: Cats are curious creatures that love to explore their surroundings.\n - Vocal: Cats communicate through a variety of vocalizations, including meows, purrs, hisses, and grow"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

Expand Down
2 changes: 1 addition & 1 deletion packages/vertexai/test-utils/mock-responses/streaming-success-basic-reply-short.txt
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
data: {"candidates": [{"content": {"parts": [{"text": "Cheyenne"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}
data: {"candidates": [{"content": {"role": "model","parts": [{"text": "Cheyenne"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE","probabilityScore": 0.02854415,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.052424565},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE","probabilityScore": 0.24926445,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.0996453},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE","probabilityScore": 0.087096825,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.043123372},{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE","probabilityScore": 0.14402841,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.086169556}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]},"usageMetadata": {"promptTokenCount": 6,"candidatesTokenCount": 4,"totalTokenCount": 10}}

2 changes: 1 addition & 1 deletion packages/vertexai/test-utils/mock-responses/streaming-success-citations.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
data: {"candidates": [{"content": {"parts": [{"text": "1. **Definition:**\nQuantum mechanics is a fundamental theory in physics that provides"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}
data: {"candidates": [{"content": {"parts": [{"text": "1. **Definition:**\nQuantum mechanics is a fundamental theory in physics that provides"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

data: {"candidates": [{"content": {"parts": [{"text": " the foundation for understanding the physical properties of nature at the scale of atoms and subatomic particles. It is based on the idea that energy, momentum, angular momentum"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

Expand Down
Loading

0 comments on commit ca41391

Please sign in to comment.