Skip to content

Commit 8e561dc

Browse files
jeremyphilemonjeremyphilemon
authored
feat: add artifact tests (#859)
1 parent 9628c54 commit 8e561dc

File tree

13 files changed

+669
-255
lines changed

13 files changed

+669
-255
lines changed

‎components/artifact-close-button.tsx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ function PureArtifactCloseButton() {
88

99
return (
1010
<Button
11+
data-testid="artifact-close-button"
1112
variant="outline"
1213
className="h-fit p-2 dark:hover:bg-zinc-700"
1314
onClick={() => {

‎components/artifact.tsx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ function PureArtifact({
269269
<AnimatePresence>
270270
{artifact.isVisible && (
271271
<motion.div
272+
data-testid="artifact"
272273
className="flex flex-row h-dvh w-dvw fixed top-0 left-0 z-50 bg-transparent"
273274
initial={{ opacity: 1 }}
274275
animate={{ opacity: 1 }}

‎components/message-actions.tsx

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import type { Message } from 'ai';
2-
import { toast } from 'sonner';
32
import { useSWRConfig } from 'swr';
43
import { useCopyToClipboard } from 'usehooks-ts';
54

@@ -15,6 +14,7 @@ import {
1514
} from './ui/tooltip';
1615
import { memo } from 'react';
1716
import equal from 'fast-deep-equal';
17+
import { toast } from 'sonner';
1818

1919
export function PureMessageActions({
2020
chatId,
@@ -57,6 +57,7 @@ export function PureMessageActions({
5757
<Tooltip>
5858
<TooltipTrigger asChild>
5959
<Button
60+
data-testid="message-upvote"
6061
className="py-1 px-2 h-fit text-muted-foreground !pointer-events-auto"
6162
disabled={vote?.isUpvoted}
6263
variant="outline"
@@ -109,6 +110,7 @@ export function PureMessageActions({
109110
<Tooltip>
110111
<TooltipTrigger asChild>
111112
<Button
113+
data-testid="message-downvote"
112114
className="py-1 px-2 h-fit text-muted-foreground !pointer-events-auto"
113115
variant="outline"
114116
disabled={vote && !vote.isUpvoted}

‎components/multimodal-input.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ function PureMultimodalInput({
247247
autoFocus
248248
onKeyDown={(event) => {
249249
if (
250-
event.key === "Enter" &&
250+
event.key === 'Enter' &&
251251
!event.shiftKey &&
252252
!event.nativeEvent.isComposing
253253
) {

‎lib/ai/models.test.ts

Lines changed: 4 additions & 214 deletions
Original file line numberDiff line numberDiff line change
@@ -1,180 +1,6 @@
1-
import { CoreMessage, FinishReason, simulateReadableStream } from 'ai';
1+
import { simulateReadableStream } from 'ai';
22
import { MockLanguageModelV1 } from 'ai/test';
3-
4-
interface ReasoningChunk {
5-
type: 'reasoning';
6-
textDelta: string;
7-
}
8-
9-
interface TextDeltaChunk {
10-
type: 'text-delta';
11-
textDelta: string;
12-
}
13-
14-
interface FinishChunk {
15-
type: 'finish';
16-
finishReason: FinishReason;
17-
logprobs: undefined;
18-
usage: { completionTokens: number; promptTokens: number };
19-
}
20-
21-
type Chunk = TextDeltaChunk | ReasoningChunk | FinishChunk;
22-
23-
const getResponseChunksByPrompt = (
24-
prompt: CoreMessage[],
25-
isReasoningEnabled: boolean = false,
26-
): Array<Chunk> => {
27-
const userMessage = prompt.at(-1);
28-
29-
if (!userMessage) {
30-
throw new Error('No user message found');
31-
}
32-
33-
if (isReasoningEnabled) {
34-
if (
35-
compareMessages(userMessage, {
36-
role: 'user',
37-
content: [{ type: 'text', text: 'why is the sky blue?' }],
38-
})
39-
) {
40-
return [
41-
{ type: 'reasoning', textDelta: 'the ' },
42-
{ type: 'reasoning', textDelta: 'sky ' },
43-
{ type: 'reasoning', textDelta: 'is ' },
44-
{ type: 'reasoning', textDelta: 'blue ' },
45-
{ type: 'reasoning', textDelta: 'because ' },
46-
{ type: 'reasoning', textDelta: 'of ' },
47-
{ type: 'reasoning', textDelta: 'rayleigh ' },
48-
{ type: 'reasoning', textDelta: 'scattering! ' },
49-
{ type: 'text-delta', textDelta: "it's " },
50-
{ type: 'text-delta', textDelta: 'just ' },
51-
{ type: 'text-delta', textDelta: 'blue ' },
52-
{ type: 'text-delta', textDelta: 'duh!' },
53-
{
54-
type: 'finish',
55-
finishReason: 'stop',
56-
logprobs: undefined,
57-
usage: { completionTokens: 10, promptTokens: 3 },
58-
},
59-
];
60-
} else if (
61-
compareMessages(userMessage, {
62-
role: 'user',
63-
content: [{ type: 'text', text: 'why is grass green?' }],
64-
})
65-
) {
66-
return [
67-
{ type: 'reasoning', textDelta: 'grass ' },
68-
{ type: 'reasoning', textDelta: 'is ' },
69-
{ type: 'reasoning', textDelta: 'green ' },
70-
{ type: 'reasoning', textDelta: 'because ' },
71-
{ type: 'reasoning', textDelta: 'of ' },
72-
{ type: 'reasoning', textDelta: 'chlorophyll ' },
73-
{ type: 'reasoning', textDelta: 'absorption! ' },
74-
{ type: 'text-delta', textDelta: "it's " },
75-
{ type: 'text-delta', textDelta: 'just ' },
76-
{ type: 'text-delta', textDelta: 'green ' },
77-
{ type: 'text-delta', textDelta: 'duh!' },
78-
{
79-
type: 'finish',
80-
finishReason: 'stop',
81-
logprobs: undefined,
82-
usage: { completionTokens: 10, promptTokens: 3 },
83-
},
84-
];
85-
}
86-
}
87-
88-
if (
89-
compareMessages(userMessage, {
90-
role: 'user',
91-
content: [{ type: 'text', text: 'why is grass green?' }],
92-
})
93-
) {
94-
return [
95-
{ type: 'text-delta', textDelta: "it's " },
96-
{ type: 'text-delta', textDelta: 'just ' },
97-
{ type: 'text-delta', textDelta: 'green ' },
98-
{ type: 'text-delta', textDelta: 'duh!' },
99-
{
100-
type: 'finish',
101-
finishReason: 'stop',
102-
logprobs: undefined,
103-
usage: { completionTokens: 10, promptTokens: 3 },
104-
},
105-
];
106-
} else if (
107-
compareMessages(userMessage, {
108-
role: 'user',
109-
content: [{ type: 'text', text: 'why is the sky blue?' }],
110-
})
111-
) {
112-
return [
113-
{ type: 'text-delta', textDelta: "it's " },
114-
{ type: 'text-delta', textDelta: 'just ' },
115-
{ type: 'text-delta', textDelta: 'blue ' },
116-
{ type: 'text-delta', textDelta: 'duh!' },
117-
{
118-
type: 'finish',
119-
finishReason: 'stop',
120-
logprobs: undefined,
121-
usage: { completionTokens: 10, promptTokens: 3 },
122-
},
123-
];
124-
} else if (
125-
compareMessages(userMessage, {
126-
role: 'user',
127-
content: [
128-
{ type: 'text', text: 'What are the advantages of using Next.js?' },
129-
],
130-
})
131-
) {
132-
return [
133-
{ type: 'text-delta', textDelta: 'with ' },
134-
{ type: 'text-delta', textDelta: 'next.js ' },
135-
{ type: 'text-delta', textDelta: 'you ' },
136-
{ type: 'text-delta', textDelta: 'can ' },
137-
{ type: 'text-delta', textDelta: 'ship ' },
138-
{ type: 'text-delta', textDelta: 'fast! ' },
139-
{
140-
type: 'finish',
141-
finishReason: 'stop',
142-
logprobs: undefined,
143-
usage: { completionTokens: 10, promptTokens: 3 },
144-
},
145-
];
146-
} else if (
147-
compareMessages(userMessage, {
148-
role: 'user',
149-
content: [
150-
{
151-
type: 'text',
152-
text: 'who painted this?',
153-
},
154-
{
155-
type: 'image',
156-
image: '...',
157-
},
158-
],
159-
})
160-
) {
161-
return [
162-
{ type: 'text-delta', textDelta: 'this ' },
163-
{ type: 'text-delta', textDelta: 'painting ' },
164-
{ type: 'text-delta', textDelta: 'is ' },
165-
{ type: 'text-delta', textDelta: 'by ' },
166-
{ type: 'text-delta', textDelta: 'monet!' },
167-
{
168-
type: 'finish',
169-
finishReason: 'stop',
170-
logprobs: undefined,
171-
usage: { completionTokens: 10, promptTokens: 3 },
172-
},
173-
];
174-
}
175-
176-
return [];
177-
};
3+
import { getResponseChunksByPrompt } from '@/tests/prompts/utils';
1784

1795
export const chatModel = new MockLanguageModelV1({
1806
doGenerate: async () => ({
@@ -236,46 +62,10 @@ export const artifactModel = new MockLanguageModelV1({
23662
usage: { promptTokens: 10, completionTokens: 20 },
23763
text: `Hello, world!`,
23864
}),
239-
doStream: async () => ({
65+
doStream: async ({ prompt }) => ({
24066
stream: simulateReadableStream({
241-
chunks: [
242-
{ type: 'text-delta', textDelta: 'test' },
243-
{
244-
type: 'finish',
245-
finishReason: 'stop',
246-
logprobs: undefined,
247-
usage: { completionTokens: 10, promptTokens: 3 },
248-
},
249-
],
67+
chunks: getResponseChunksByPrompt(prompt),
25068
}),
25169
rawCall: { rawPrompt: null, rawSettings: {} },
25270
}),
25371
});
254-
255-
function compareMessages(msg1: CoreMessage, msg2: CoreMessage): boolean {
256-
if (msg1.role !== msg2.role) return false;
257-
258-
if (!Array.isArray(msg1.content) || !Array.isArray(msg2.content)) {
259-
return false;
260-
}
261-
262-
if (msg1.content.length !== msg2.content.length) return false;
263-
264-
for (let i = 0; i < msg1.content.length; i++) {
265-
const item1 = msg1.content[i];
266-
const item2 = msg2.content[i];
267-
268-
if (item1.type !== item2.type) return false;
269-
270-
if (item1.type === 'image' && item2.type === 'image') {
271-
// if (item1.image.toString() !== item2.image.toString()) return false;
272-
// if (item1.mimeType !== item2.mimeType) return false;
273-
} else if (item1.type === 'text' && item2.type === 'text') {
274-
if (item1.text !== item2.text) return false;
275-
} else {
276-
return false;
277-
}
278-
}
279-
280-
return true;
281-
}

‎playwright.config.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,15 @@ export default defineConfig({
8282
storageState: 'playwright/.reasoning/session.json',
8383
},
8484
},
85+
{
86+
name: 'artifacts',
87+
testMatch: /artifacts.test.ts/,
88+
dependencies: ['setup:auth'],
89+
use: {
90+
...devices['Desktop Chrome'],
91+
storageState: 'playwright/.auth/session.json',
92+
},
93+
},
8594

8695
// {
8796
// name: 'firefox',

‎tests/artifacts.test.ts

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { expect, test } from '@playwright/test';
2+
import { ChatPage } from './pages/chat';
3+
import { ArtifactPage } from './pages/artifact';
4+
5+
test.describe('artifacts activity', () => {
6+
let chatPage: ChatPage;
7+
let artifactPage: ArtifactPage;
8+
9+
test.beforeEach(async ({ page }) => {
10+
chatPage = new ChatPage(page);
11+
artifactPage = new ArtifactPage(page);
12+
13+
await chatPage.createNewChat();
14+
});
15+
16+
test('create a text artifact', async () => {
17+
await chatPage.createNewChat();
18+
19+
await chatPage.sendUserMessage(
20+
'Help me write an essay about Silicon Valley',
21+
);
22+
await artifactPage.isGenerationComplete();
23+
24+
expect(artifactPage.artifact).toBeVisible();
25+
26+
const assistantMessage = await chatPage.getRecentAssistantMessage();
27+
expect(assistantMessage.content).toBe(
28+
'A document was created and is now visible to the user.',
29+
);
30+
31+
await chatPage.hasChatIdInUrl();
32+
});
33+
34+
test('toggle artifact visibility', async () => {
35+
await chatPage.createNewChat();
36+
37+
await chatPage.sendUserMessage(
38+
'Help me write an essay about Silicon Valley',
39+
);
40+
await artifactPage.isGenerationComplete();
41+
42+
expect(artifactPage.artifact).toBeVisible();
43+
44+
const assistantMessage = await chatPage.getRecentAssistantMessage();
45+
expect(assistantMessage.content).toBe(
46+
'A document was created and is now visible to the user.',
47+
);
48+
49+
await artifactPage.closeArtifact();
50+
await chatPage.isElementNotVisible('artifact');
51+
});
52+
53+
test('send follow up message after generation', async () => {
54+
await chatPage.createNewChat();
55+
56+
await chatPage.sendUserMessage(
57+
'Help me write an essay about Silicon Valley',
58+
);
59+
await artifactPage.isGenerationComplete();
60+
61+
expect(artifactPage.artifact).toBeVisible();
62+
63+
const assistantMessage = await artifactPage.getRecentAssistantMessage();
64+
expect(assistantMessage.content).toBe(
65+
'A document was created and is now visible to the user.',
66+
);
67+
68+
await artifactPage.sendUserMessage('Thanks!');
69+
await artifactPage.isGenerationComplete();
70+
71+
const secondAssistantMessage = await chatPage.getRecentAssistantMessage();
72+
expect(secondAssistantMessage.content).toBe("You're welcome!");
73+
});
74+
});

0 commit comments

Comments
 (0)