From 5985ec02b79af0b15ab83f25f68d7a101c49bea4 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Mon, 27 May 2024 15:52:03 +0200 Subject: [PATCH 1/9] feat(logs): pagination and infinite scroll --- .env.example | 3 + package-lock.json | 209 ++++++++++++++++++ packages/logs/lib/client.integration.test.ts | 10 + .../lib/models/messages.integration.test.ts | 57 +++++ packages/logs/lib/models/messages.ts | 10 +- .../logs/searchOperations.integration.test.ts | 4 +- .../controllers/v1/logs/searchOperations.ts | 8 +- packages/types/lib/logs/api.ts | 3 +- packages/webapp/package.json | 1 + packages/webapp/src/components/DebugMode.tsx | 2 +- packages/webapp/src/components/LeftNavBar.tsx | 4 +- packages/webapp/src/components/TopNavBar.tsx | 64 +++--- .../webapp/src/components/ui/ScrollArea.tsx | 36 +++ packages/webapp/src/hooks/useLogs.tsx | 8 +- .../webapp/src/layout/DashboardLayout.tsx | 25 ++- packages/webapp/src/pages/Activity.tsx | 6 +- .../webapp/src/pages/EnvironmentSettings.tsx | 2 +- packages/webapp/src/pages/Logs/Search.tsx | 78 ++++++- .../src/pages/Logs/components/DatePicker.tsx | 27 ++- 19 files changed, 482 insertions(+), 75 deletions(-) create mode 100644 packages/logs/lib/models/messages.integration.test.ts create mode 100644 packages/webapp/src/components/ui/ScrollArea.tsx diff --git a/.env.example b/.env.example index 7bb7766df0..3034cb4eb2 100644 --- a/.env.example +++ b/.env.example @@ -95,3 +95,6 @@ GOOGLE_APPLICATION_CREDENTIALS= # Key to send email MAILGUN_API_KEY= + +# Redis (optional) +NANGO_REDIS_URL= diff --git a/package-lock.json b/package-lock.json index 8b68a3dc8e..5e4678e461 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34945,6 +34945,7 @@ "@radix-ui/react-dropdown-menu": "2.0.6", "@radix-ui/react-icons": "1.3.0", "@radix-ui/react-popover": "1.0.7", + "@radix-ui/react-scroll-area": "1.0.5", "@radix-ui/react-tooltip": "1.0.7", "@sentry/react": "8.4.0", "@tailwindcss/forms": "0.5.3", @@ -35108,6 +35109,214 @@ "react": "^18.2.0" } }, + "packages/webapp/node_modules/@radix-ui/number": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.0.1.tgz", + "integrity": "sha512-T5gIdVO2mmPW3NNhjNgEP3cqMXjXL9UbO0BzWcXfvdBs+BohbQxvd/K5hSVKmn9/lbTdsQVKbUcP5WLCwvUbBg==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10" + } + }, + "packages/webapp/node_modules/@radix-ui/primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.0.1.tgz", + "integrity": "sha512-yQ8oGX2GVsEYMWGxcovu1uGWPCxV5BFfeeYxqPmuAzUyLT9qmaMXSAhXpb0WrspIeqYzdJpkh2vHModJPgRIaw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10" + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.0.5.tgz", + "integrity": "sha512-b6PAgH4GQf9QEn8zbT2XUHpW5z8BzqEc7Kl11TwDrvuTrxlkcjTD5qa/bxgKr+nmuXKu4L/W5UZ4mlP/VG/5Gw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/number": "1.0.1", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-presence": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-callback-ref": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-compose-refs": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz", + "integrity": "sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-context": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.0.1.tgz", + "integrity": "sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-direction": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.0.1.tgz", + "integrity": "sha512-RXcvnXgyvYvBEOhCBuddKecVkoMiI10Jcm5cTI7abJRAHYfFxeu+FBQs/DvdxSYucxR5mna0dNsL6QFlds5TMA==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-presence": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz", + "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-compose-refs": "1.0.1", + "@radix-ui/react-use-layout-effect": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-primitive": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz", + "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-slot": "1.0.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-primitive/node_modules/@radix-ui/react-slot": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz", + "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/react-compose-refs": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz", + "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "packages/webapp/node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz", + "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.13.10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "packages/webapp/node_modules/@remix-run/router": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.3.3.tgz", diff --git a/packages/logs/lib/client.integration.test.ts b/packages/logs/lib/client.integration.test.ts index d093f4fdbb..39becdb483 100644 --- a/packages/logs/lib/client.integration.test.ts +++ b/packages/logs/lib/client.integration.test.ts @@ -27,6 +27,15 @@ describe('client', () => { vi.clearAllMocks(); }); + it('should list nothing', async () => { + const list = await listOperations({ accountId: account.id, limit: 1, states: ['all'] }); + expect(list).toStrictEqual({ + cursor: null, + count: 0, + items: [] + }); + }); + it('should insert an operation', async () => { const spy = vi.spyOn(model, 'createMessage'); const ctx = await logContextGetter.create(operationPayload, { start: false, account, environment }, { logToConsole: false }); @@ -35,6 +44,7 @@ describe('client', () => { const list = await listOperations({ accountId: account.id, limit: 1, states: ['all'] }); expect(list).toStrictEqual({ + cursor: expect.any(String), count: 1, items: [ { diff --git a/packages/logs/lib/models/messages.integration.test.ts b/packages/logs/lib/models/messages.integration.test.ts new file mode 100644 index 0000000000..a6cc618ac6 --- /dev/null +++ b/packages/logs/lib/models/messages.integration.test.ts @@ -0,0 +1,57 @@ +import { describe, beforeAll, it, expect, vi } from 'vitest'; +import { deleteIndex, migrateMapping } from '../es/helpers.js'; +import type { ListOperations } from './messages.js'; +import { listOperations } from './messages.js'; +import { afterEach } from 'node:test'; +import { logContextGetter } from './logContextGetter.js'; +import type { OperationRowInsert } from '@nangohq/types'; + +const account = { id: 1234, name: 'test' }; +const environment = { id: 5678, name: 'dev' }; +const operationPayload: OperationRowInsert = { operation: { type: 'sync', action: 'run' }, message: '' }; + +describe('model', () => { + beforeAll(async () => { + await deleteIndex(); + await migrateMapping(); + }); + afterEach(() => { + vi.clearAllMocks(); + }); + + describe('operations', () => { + it('should list nothing', async () => { + const list = await listOperations({ accountId: account.id, limit: 1, states: ['all'] }); + expect(list).toStrictEqual({ + cursor: null, + count: 0, + items: [] + }); + }); + + it('should pagination', async () => { + await logContextGetter.create(operationPayload, { start: false, account, environment }, { logToConsole: false }); + await logContextGetter.create(operationPayload, { start: false, account, environment }, { logToConsole: false }); + + // First operation + const list1 = await listOperations({ accountId: account.id, limit: 1, states: ['all'] }); + expect(list1.count).toBe(2); + expect(list1.items).toHaveLength(1); + expect(list1.cursor).toBeDefined(); + + // Second operation + const list2 = await listOperations({ accountId: account.id, limit: 1, states: ['all'], cursor: list1.cursor! }); + expect(list2.count).toBe(2); + expect(list2.items).toHaveLength(1); + expect(list2.cursor).toBeDefined(); + expect(list2.items[0]!.id).not.toEqual(list1.items[0]!.id); + + // Empty results + // When we get the second operation, it's not possible to know if there are more after so we still need to return a cursor + const list3 = await listOperations({ accountId: account.id, limit: 1, states: ['all'], cursor: list2.cursor! }); + expect(list3.count).toBe(2); + expect(list3.items).toHaveLength(0); + expect(list3.cursor).toBeNull(); + }); + }); +}); diff --git a/packages/logs/lib/models/messages.ts b/packages/logs/lib/models/messages.ts index de92fabf61..4d46ce2c02 100644 --- a/packages/logs/lib/models/messages.ts +++ b/packages/logs/lib/models/messages.ts @@ -16,6 +16,7 @@ import { errors } from '@elastic/elasticsearch'; export interface ListOperations { count: number; items: OperationRow[]; + cursor: string | null; } export interface ListMessages { count: number; @@ -52,6 +53,7 @@ export async function listOperations(opts: { connections?: SearchOperationsConnection[] | undefined; syncs?: SearchOperationsSync[] | undefined; period?: SearchOperationsPeriod | undefined; + cursor?: string | null | undefined; }): Promise { const query: estypes.QueryDslQueryContainer = { bool: { @@ -133,15 +135,19 @@ export async function listOperations(opts: { size: opts.limit, sort: [{ createdAt: 'desc' }, '_score'], track_total_hits: true, + search_after: opts.cursor ? JSON.parse(Buffer.from(opts.cursor, 'base64').toString('utf8')) : undefined, query }); const hits = res.hits; + console.log('top', { first: hits.hits[0], last: hits.hits[hits.hits.length - 1] }); + const total = typeof hits.total === 'object' ? hits.total.value : hits.hits.length; return { - count: typeof hits.total === 'object' ? hits.total.value : hits.hits.length, + count: total, items: hits.hits.map((hit) => { return hit._source!; - }) + }), + cursor: hits.hits.length > 0 && total > hits.hits.length ? Buffer.from(JSON.stringify(hits.hits[hits.hits.length - 1]!.sort)).toString('base64') : null }; } diff --git a/packages/server/lib/controllers/v1/logs/searchOperations.integration.test.ts b/packages/server/lib/controllers/v1/logs/searchOperations.integration.test.ts index 4bc77d11cd..3ae12ebcec 100644 --- a/packages/server/lib/controllers/v1/logs/searchOperations.integration.test.ts +++ b/packages/server/lib/controllers/v1/logs/searchOperations.integration.test.ts @@ -135,7 +135,7 @@ describe('POST /logs/operations', () => { userId: null } ], - pagination: { total: 1 } + pagination: { total: 1, cursor: expect.any(String) } }); }); @@ -161,7 +161,7 @@ describe('POST /logs/operations', () => { expect(res.res.status).toBe(200); expect(res.json).toStrictEqual({ data: [], - pagination: { total: 0 } + pagination: { total: 0, cursor: expect.any(String) } }); }); }); diff --git a/packages/server/lib/controllers/v1/logs/searchOperations.ts b/packages/server/lib/controllers/v1/logs/searchOperations.ts index eaba64f867..89680f8916 100644 --- a/packages/server/lib/controllers/v1/logs/searchOperations.ts +++ b/packages/server/lib/controllers/v1/logs/searchOperations.ts @@ -36,7 +36,8 @@ const validation = z integrations: z.array(z.string()).optional().default(['all']), connections: z.array(z.string()).optional().default(['all']), syncs: z.array(z.string()).optional().default(['all']), - period: z.object({ from: z.string().datetime(), to: z.string().datetime() }).optional() + period: z.object({ from: z.string().datetime(), to: z.string().datetime() }).optional(), + cursor: z.string().or(z.null()).optional() }) .strict(); @@ -71,11 +72,12 @@ export const searchOperations = asyncWrapper(async (req, res) integrations: body.integrations, connections: body.connections, syncs: body.syncs, - period: body.period + period: body.period, + cursor: body.cursor }); res.status(200).send({ data: rawOps.items, - pagination: { total: rawOps.count } + pagination: { total: rawOps.count, cursor: rawOps.cursor } }); }); diff --git a/packages/types/lib/logs/api.ts b/packages/types/lib/logs/api.ts index a73265290a..74957729ac 100644 --- a/packages/types/lib/logs/api.ts +++ b/packages/types/lib/logs/api.ts @@ -16,10 +16,11 @@ export type SearchOperations = Endpoint<{ connections?: SearchOperationsConnection[] | undefined; syncs?: SearchOperationsSync[] | undefined; period?: SearchOperationsPeriod | undefined; + cursor?: string | null | undefined; }; Success: { data: OperationRow[]; - pagination: { total: number }; + pagination: { total: number; cursor: string | null }; }; }>; export type SearchOperationsState = 'all' | MessageState; diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 25de145d24..1351679c78 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -42,6 +42,7 @@ "@radix-ui/react-dropdown-menu": "2.0.6", "@radix-ui/react-icons": "1.3.0", "@radix-ui/react-popover": "1.0.7", + "@radix-ui/react-scroll-area": "1.0.5", "@radix-ui/react-tooltip": "1.0.7", "@sentry/react": "8.4.0", "@tailwindcss/forms": "0.5.3", diff --git a/packages/webapp/src/components/DebugMode.tsx b/packages/webapp/src/components/DebugMode.tsx index 5b62ab1e25..17baf0d14b 100644 --- a/packages/webapp/src/components/DebugMode.tsx +++ b/packages/webapp/src/components/DebugMode.tsx @@ -7,5 +7,5 @@ export const DebugMode: React.FC = () => { return null; } - return
Debug mode activated
; + return
Debug mode activated
; }; diff --git a/packages/webapp/src/components/LeftNavBar.tsx b/packages/webapp/src/components/LeftNavBar.tsx index c93499b4dd..8b6a88faf9 100644 --- a/packages/webapp/src/components/LeftNavBar.tsx +++ b/packages/webapp/src/components/LeftNavBar.tsx @@ -89,8 +89,8 @@ export default function LeftNavBar(props: LeftNavBarProps) { } return ( -
-
+
+
Nango diff --git a/packages/webapp/src/components/TopNavBar.tsx b/packages/webapp/src/components/TopNavBar.tsx index 61f5748061..7ccd67c15f 100644 --- a/packages/webapp/src/components/TopNavBar.tsx +++ b/packages/webapp/src/components/TopNavBar.tsx @@ -16,39 +16,37 @@ export default function NavBar() { }; return ( -
-
-
- {isHNDemo && ( - - This is a test account. Click{' '} - {' '} - to create a real account. - - )} -
- +
+
+ {isHNDemo && ( + + This is a test account. Click{' '} + {' '} + to create a real account. + + )} +
+
); diff --git a/packages/webapp/src/components/ui/ScrollArea.tsx b/packages/webapp/src/components/ui/ScrollArea.tsx new file mode 100644 index 0000000000..d0cb006d64 --- /dev/null +++ b/packages/webapp/src/components/ui/ScrollArea.tsx @@ -0,0 +1,36 @@ +import * as React from 'react'; +import * as ScrollAreaPrimitive from '@radix-ui/react-scroll-area'; +import { cn } from '../../utils/utils'; + +const ScrollArea = React.forwardRef, React.ComponentPropsWithoutRef>( + ({ className, children, ...props }, ref) => ( + + {children} + + + + ) +); +ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName; + +const ScrollBar = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, orientation = 'vertical', ...props }, ref) => ( + + + +)); +ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName; + +export { ScrollArea, ScrollBar }; diff --git a/packages/webapp/src/hooks/useLogs.tsx b/packages/webapp/src/hooks/useLogs.tsx index b18683696a..5cfe446c62 100644 --- a/packages/webapp/src/hooks/useLogs.tsx +++ b/packages/webapp/src/hooks/useLogs.tsx @@ -8,12 +8,12 @@ export function useSearchOperations(enabled: boolean, env: string, body: SearchO const [data, setData] = useState(); const [error, setError] = useState(); - async function fetchData() { + async function fetchData(cursor?: SearchOperations['Body']['cursor']) { setLoading(true); try { const res = await fetch(`/api/v1/logs/operations?env=${env}`, { method: 'POST', - body: JSON.stringify(body), + body: JSON.stringify({ ...body, cursor }), headers: { 'Content-Type': 'application/json' } }); if (res.status !== 200) { @@ -38,9 +38,9 @@ export function useSearchOperations(enabled: boolean, env: string, body: SearchO } }, [enabled, env, body.limit, body.states, body.integrations, body.period, body.types, body.connections, body.syncs]); - function trigger() { + function trigger(cursor?: SearchOperations['Body']['cursor']) { if (enabled && !loading) { - void fetchData(); + void fetchData(cursor); } } diff --git a/packages/webapp/src/layout/DashboardLayout.tsx b/packages/webapp/src/layout/DashboardLayout.tsx index e7caf9147b..c26c8f1cb7 100644 --- a/packages/webapp/src/layout/DashboardLayout.tsx +++ b/packages/webapp/src/layout/DashboardLayout.tsx @@ -1,23 +1,32 @@ +import type { ClassValue } from 'clsx'; import { DebugMode } from '../components/DebugMode'; import type { LeftNavBarItems } from '../components/LeftNavBar'; import LeftNavBar from '../components/LeftNavBar'; import TopNavBar from '../components/TopNavBar'; +import { cn } from '../utils/utils'; interface DashboardLayoutI { children: React.ReactNode; selectedItem: LeftNavBarItems; - marginBottom?: number; + fullWidth?: boolean; + className?: ClassValue; } -export default function DashboardLayout({ children, selectedItem, marginBottom = 24 }: DashboardLayoutI) { +export default function DashboardLayout({ children, selectedItem, fullWidth = false, className }: DashboardLayoutI) { return ( -
- - -
+
+
+ +
+
-
-
{children}
+
+
+
+ +
+
+
{children}
diff --git a/packages/webapp/src/pages/Activity.tsx b/packages/webapp/src/pages/Activity.tsx index 9562fa2943..c378e0ccf0 100644 --- a/packages/webapp/src/pages/Activity.tsx +++ b/packages/webapp/src/pages/Activity.tsx @@ -363,7 +363,7 @@ export default function Activity() { if (error || logActivitiesError) { return ( - + ); @@ -371,14 +371,14 @@ export default function Activity() { if (!activities) { return ( - + ); } return ( - +
diff --git a/packages/webapp/src/pages/EnvironmentSettings.tsx b/packages/webapp/src/pages/EnvironmentSettings.tsx index 41ef230c39..41a1fac0a2 100644 --- a/packages/webapp/src/pages/EnvironmentSettings.tsx +++ b/packages/webapp/src/pages/EnvironmentSettings.tsx @@ -442,7 +442,7 @@ export const EnvironmentSettings: React.FC = () => { {secretKey && ( -
+

Environment Settings

diff --git a/packages/webapp/src/pages/Logs/Search.tsx b/packages/webapp/src/pages/Logs/Search.tsx index 35ef78845a..9692c0fb77 100644 --- a/packages/webapp/src/pages/Logs/Search.tsx +++ b/packages/webapp/src/pages/Logs/Search.tsx @@ -9,18 +9,26 @@ import { getCoreRowModel, useReactTable, flexRender } from '@tanstack/react-tabl import { MultiSelect } from './components/MultiSelect'; import { columns, integrationsDefaultOptions, statusDefaultOptions, statusOptions, syncsDefaultOptions, typesDefaultOptions } from './constants'; -import { useEffect, useMemo, useState } from 'react'; -import type { SearchOperationsIntegration, SearchOperationsPeriod, SearchOperationsState, SearchOperationsSync, SearchOperationsType } from '@nangohq/types'; +import { useEffect, useMemo, useRef, useState } from 'react'; +import type { + SearchOperationsData, + SearchOperationsIntegration, + SearchOperationsPeriod, + SearchOperationsState, + SearchOperationsSync, + SearchOperationsType +} from '@nangohq/types'; import Spinner from '../../components/ui/Spinner'; -import { OperationRow } from './components/OperationRow'; // import { Input } from '../../components/ui/input/Input'; // import { MagnifyingGlassIcon } from '@radix-ui/react-icons'; import { formatQuantity } from '../../utils/utils'; import { Link, useSearchParams } from 'react-router-dom'; -import { useInterval } from 'react-use'; +import { useIntersection, useInterval } from 'react-use'; import { SearchableMultiSelect } from './components/SearchableMultiSelect'; import { TypesSelect } from './components/TypesSelect'; import { DatePicker } from './components/DatePicker'; +import Button from '../../components/ui/button/Button'; +import { OperationRow } from './components/OperationRow'; export const LogsSearch: React.FC = () => { const env = useStore((state) => state.env); @@ -37,10 +45,20 @@ export const LogsSearch: React.FC = () => { const [connections, setConnections] = useState(integrationsDefaultOptions); const [syncs, setSyncs] = useState(syncsDefaultOptions); const [period, setPeriod] = useState(); - const { data, error, loading, trigger } = useSearchOperations(synced, env, { limit: 20, states, types, integrations, connections, syncs, period }); + const [cursor, setCursor] = useState(); + const { data, error, loading, trigger } = useSearchOperations(synced, env, { limit: 5, states, types, integrations, connections, syncs, period }); + const [operations, setOperations] = useState([]); + + // Infinite scroll + const bottomScrollRef = useRef(null); + const bottomScroll = useIntersection(bottomScrollRef, { + root: null, + rootMargin: '0px', + threshold: 1 + }); const table = useReactTable({ - data: data ? data.data : [], + data: operations, columns, getCoreRowModel: getCoreRowModel() }); @@ -76,6 +94,11 @@ export const LogsSearch: React.FC = () => { const tmpTo = searchParams.get('to'); setPeriod(tmpFrom && tmpTo ? { from: tmpFrom, to: tmpTo } : undefined); + const tmpCursor = searchParams.get('cursor'); + if (tmpCursor) { + setCursor(tmpCursor); + } + setSynced(true); }, [searchParams, synced] @@ -95,9 +118,12 @@ export const LogsSearch: React.FC = () => { tmp.set('from', period.from); tmp.set('to', period.to); } + if (cursor) { + tmp.set('cursor', cursor); + } setSearchParams(tmp); }, - [states, integrations, period, connections, syncs, types] + [states, integrations, period, connections, syncs, types, cursor] ); useEffect(() => { @@ -123,9 +149,34 @@ export const LogsSearch: React.FC = () => { return formatQuantity(data.pagination.total); }, [data?.pagination]); + useEffect(() => { + if (!bottomScroll) { + return; + } + console.log('hello..', bottomScroll); + if (!bottomScroll.isIntersecting) { + return; + } + if (!data?.pagination.cursor || loading) { + return; + } + console.log('trigger search'); + setCursor(data.pagination.cursor); + }, [bottomScroll, data]); + + const loadMore = () => { + if (data?.pagination.cursor) { + trigger(data.pagination.cursor); + } + }; + + useEffect(() => { + setOperations((prev) => [...prev, ...(data?.data || [])]); + }, [data]); + if (error) { return ( - +

Logs

{error.error.code === 'feature_disabled' ? (
@@ -149,7 +200,7 @@ export const LogsSearch: React.FC = () => { if ((loading && !data) || !data) { return ( - + ); @@ -157,7 +208,7 @@ export const LogsSearch: React.FC = () => { if (data.pagination.total <= 0 && !hasLogs) { return ( - +

Logs

@@ -169,7 +220,7 @@ export const LogsSearch: React.FC = () => { } return ( - +

Logs {loading && }

{total} logs found
@@ -222,6 +273,11 @@ export const LogsSearch: React.FC = () => { )} + {data.pagination.total > 0 && data.data.length > 0 && ( +
+ +
+ )} ); }; diff --git a/packages/webapp/src/pages/Logs/components/DatePicker.tsx b/packages/webapp/src/pages/Logs/components/DatePicker.tsx index 5b8d8d1527..ac8a3bb185 100644 --- a/packages/webapp/src/pages/Logs/components/DatePicker.tsx +++ b/packages/webapp/src/pages/Logs/components/DatePicker.tsx @@ -57,19 +57,23 @@ export const DatePicker: React.FC<{ const [selectedPreset, setSelectedPreset] = useState(undefined); const [date, setDate] = useState(); + const [tmpDate, setTmpDate] = useState(); const months = useMemo(() => { const today = new Date(); return today.getDate() < 24 ? 2 : 1; }, []); + const disabledBefore = useMemo(() => { return addDays(new Date(), -14); }, []); + const disabledAfter = useMemo(() => { return new Date(); }, []); + const display = useMemo(() => { - if (!date || !date.from) { + if (!date || !date.from || !date.to) { return 'Live - Last 14 days'; } if (date.from && date.to) { @@ -82,20 +86,29 @@ export const DatePicker: React.FC<{ const range = getPresetRange(preset); setSelectedPreset(preset); onChange(range); + setTmpDate(range); }; const onClickLive = () => { setSelectedPreset(undefined); onChange(undefined); + setTmpDate(undefined); }; useEffect(() => { setDate(period ? { from: new Date(period.from), to: new Date(period.to) } : undefined); }, [period]); + useEffect(() => { + // We use a tmp date because we only want to commit full range, not partial from/to + if (!tmpDate || (tmpDate.from && tmpDate.to)) { + onChange(tmpDate); + } + }, [tmpDate]); + return ( - + +
)} From 533c4f45618173df57938e693423348a12609842 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 28 May 2024 10:28:42 +0200 Subject: [PATCH 3/9] review --- packages/logs/lib/client.integration.test.ts | 2 +- packages/logs/lib/models/helpers.ts | 9 +++++++++ .../logs/lib/models/messages.integration.test.ts | 6 +++--- packages/logs/lib/models/messages.ts | 13 ++++++------- .../v1/logs/searchOperations.integration.test.ts | 8 ++++---- .../lib/controllers/v1/logs/searchOperations.ts | 10 ++++++---- packages/webapp/src/pages/Logs/Search.tsx | 2 +- 7 files changed, 30 insertions(+), 20 deletions(-) diff --git a/packages/logs/lib/client.integration.test.ts b/packages/logs/lib/client.integration.test.ts index 39becdb483..32738c74ec 100644 --- a/packages/logs/lib/client.integration.test.ts +++ b/packages/logs/lib/client.integration.test.ts @@ -44,7 +44,7 @@ describe('client', () => { const list = await listOperations({ accountId: account.id, limit: 1, states: ['all'] }); expect(list).toStrictEqual({ - cursor: expect.any(String), + cursor: null, count: 1, items: [ { diff --git a/packages/logs/lib/models/helpers.ts b/packages/logs/lib/models/helpers.ts index 66718bc9a6..af6df5a96f 100644 --- a/packages/logs/lib/models/helpers.ts +++ b/packages/logs/lib/models/helpers.ts @@ -1,6 +1,7 @@ import { nanoid } from '@nangohq/utils'; import type { MessageRow } from '@nangohq/types'; import { z } from 'zod'; +import type { estypes } from '@elastic/elasticsearch'; export const operationIdRegex = z.string().regex(/([0-9]|[a-zA-Z0-9]{20})/); @@ -72,3 +73,11 @@ export const oldLevelToNewLevel = { silly: 'debug', http: 'info' } as const; + +export function createCursor({ sort }: estypes.SearchHit): string { + return Buffer.from(JSON.stringify(sort)).toString('base64'); +} + +export function parseCursor(str: string): any[] { + return JSON.parse(Buffer.from(str, 'base64').toString('utf8')); +} diff --git a/packages/logs/lib/models/messages.integration.test.ts b/packages/logs/lib/models/messages.integration.test.ts index a6cc618ac6..a9e616d495 100644 --- a/packages/logs/lib/models/messages.integration.test.ts +++ b/packages/logs/lib/models/messages.integration.test.ts @@ -29,17 +29,17 @@ describe('model', () => { }); }); - it('should pagination', async () => { + it('should paginate', async () => { await logContextGetter.create(operationPayload, { start: false, account, environment }, { logToConsole: false }); await logContextGetter.create(operationPayload, { start: false, account, environment }, { logToConsole: false }); - // First operation + // First operation = should list one const list1 = await listOperations({ accountId: account.id, limit: 1, states: ['all'] }); expect(list1.count).toBe(2); expect(list1.items).toHaveLength(1); expect(list1.cursor).toBeDefined(); - // Second operation + // Second operation = should list the second one const list2 = await listOperations({ accountId: account.id, limit: 1, states: ['all'], cursor: list1.cursor! }); expect(list2.count).toBe(2); expect(list2.items).toHaveLength(1); diff --git a/packages/logs/lib/models/messages.ts b/packages/logs/lib/models/messages.ts index b226ff48bd..c74a1856a3 100644 --- a/packages/logs/lib/models/messages.ts +++ b/packages/logs/lib/models/messages.ts @@ -12,6 +12,8 @@ import type { import { indexMessages } from '../es/schema.js'; import type { estypes } from '@elastic/elasticsearch'; import { errors } from '@elastic/elasticsearch'; +import { createCursor, parseCursor } from './helpers.js'; +import { isTest } from '@nangohq/utils'; export interface ListOperations { count: number; @@ -36,7 +38,7 @@ export async function createMessage(row: MessageRow): Promise { index: indexMessages.index, id: row.id, document: row, - refresh: true + refresh: isTest }); } @@ -130,7 +132,7 @@ export async function listOperations(opts: { }); } - const cursor = opts.cursor ? JSON.parse(Buffer.from(opts.cursor, 'base64').toString('utf8')) : undefined; + const cursor = opts.cursor ? parseCursor(opts.cursor) : undefined; const res = await client.search({ index: indexMessages.index, size: opts.limit, @@ -147,10 +149,7 @@ export async function listOperations(opts: { items: hits.hits.map((hit) => { return hit._source!; }), - cursor: - hits.hits.length > 0 && total > hits.hits.length && hits.hits.length >= opts.limit - ? Buffer.from(JSON.stringify(hits.hits[hits.hits.length - 1]!.sort)).toString('base64') - : null + cursor: hits.hits.length > 0 && total > hits.hits.length && hits.hits.length >= opts.limit ? createCursor(hits.hits[hits.hits.length - 1]!) : null }; } @@ -172,7 +171,7 @@ export async function update(opts: { id: MessageRow['id']; data: Partial>>({ index: indexMessages.index, id: opts.id, - refresh: true, + refresh: isTest, body: { doc: { ...opts.data, diff --git a/packages/server/lib/controllers/v1/logs/searchOperations.integration.test.ts b/packages/server/lib/controllers/v1/logs/searchOperations.integration.test.ts index 3ae12ebcec..674a533672 100644 --- a/packages/server/lib/controllers/v1/logs/searchOperations.integration.test.ts +++ b/packages/server/lib/controllers/v1/logs/searchOperations.integration.test.ts @@ -71,9 +71,9 @@ describe('POST /logs/operations', () => { isSuccess(res.json); expect(res.res.status).toBe(200); - expect(res.json).toStrictEqual({ + expect(res.json).toStrictEqual({ data: [], - pagination: { total: 0 } + pagination: { total: 0, cursor: null } }); }); @@ -135,7 +135,7 @@ describe('POST /logs/operations', () => { userId: null } ], - pagination: { total: 1, cursor: expect.any(String) } + pagination: { total: 1, cursor: null } }); }); @@ -161,7 +161,7 @@ describe('POST /logs/operations', () => { expect(res.res.status).toBe(200); expect(res.json).toStrictEqual({ data: [], - pagination: { total: 0, cursor: expect.any(String) } + pagination: { total: 0, cursor: null } }); }); }); diff --git a/packages/server/lib/controllers/v1/logs/searchOperations.ts b/packages/server/lib/controllers/v1/logs/searchOperations.ts index 89680f8916..693ac4b796 100644 --- a/packages/server/lib/controllers/v1/logs/searchOperations.ts +++ b/packages/server/lib/controllers/v1/logs/searchOperations.ts @@ -6,9 +6,10 @@ import { model, envs } from '@nangohq/logs'; const validation = z .object({ - limit: z.number().optional().default(100), + limit: z.number().max(500).optional().default(100), states: z .array(z.enum(['all', 'waiting', 'running', 'success', 'failed', 'timeout', 'cancelled'])) + .max(10) .optional() .default(['all']), types: z @@ -31,11 +32,12 @@ const validation = z 'webhook' ]) ) + .max(20) .optional() .default(['all']), - integrations: z.array(z.string()).optional().default(['all']), - connections: z.array(z.string()).optional().default(['all']), - syncs: z.array(z.string()).optional().default(['all']), + integrations: z.array(z.string()).max(10).optional().default(['all']), + connections: z.array(z.string()).max(10).optional().default(['all']), + syncs: z.array(z.string()).max(10).optional().default(['all']), period: z.object({ from: z.string().datetime(), to: z.string().datetime() }).optional(), cursor: z.string().or(z.null()).optional() }) diff --git a/packages/webapp/src/pages/Logs/Search.tsx b/packages/webapp/src/pages/Logs/Search.tsx index 68834c33b8..b2de0cee34 100644 --- a/packages/webapp/src/pages/Logs/Search.tsx +++ b/packages/webapp/src/pages/Logs/Search.tsx @@ -306,7 +306,7 @@ export const LogsSearch: React.FC = () => { ) : operations.length <= 0 && !loading && readyToDisplay ? ( - No results. {loading && 'true'} + No results. ) : ( From e216e4569f2ddc6976b95538c0d7d4e0ba4d01ef Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 28 May 2024 10:33:37 +0200 Subject: [PATCH 4/9] feat(logs): messages infinite scroll and live refresh --- .../lib/models/messages.integration.test.ts | 66 +++++++- packages/logs/lib/models/messages.ts | 39 ++++- .../logs/searchMessages.integration.test.ts | 4 +- .../lib/controllers/v1/logs/searchMessages.ts | 15 +- packages/types/lib/logs/api.ts | 11 +- packages/webapp/src/components/ui/Drawer.tsx | 5 +- packages/webapp/src/components/ui/Table.tsx | 4 +- packages/webapp/src/hooks/useLogs.tsx | 49 ++++-- .../webapp/src/pages/Logs/ShowMessage.tsx | 4 +- .../webapp/src/pages/Logs/ShowOperation.tsx | 4 +- .../src/pages/Logs/components/MessageRow.tsx | 6 +- .../pages/Logs/components/OperationRow.tsx | 2 +- .../Logs/components/SearchInOperation.tsx | 145 +++++++++++++++--- 13 files changed, 290 insertions(+), 64 deletions(-) diff --git a/packages/logs/lib/models/messages.integration.test.ts b/packages/logs/lib/models/messages.integration.test.ts index a9e616d495..0d87fe6c7f 100644 --- a/packages/logs/lib/models/messages.integration.test.ts +++ b/packages/logs/lib/models/messages.integration.test.ts @@ -1,7 +1,7 @@ import { describe, beforeAll, it, expect, vi } from 'vitest'; import { deleteIndex, migrateMapping } from '../es/helpers.js'; -import type { ListOperations } from './messages.js'; -import { listOperations } from './messages.js'; +import type { ListMessages, ListOperations } from './messages.js'; +import { listMessages, listOperations } from './messages.js'; import { afterEach } from 'node:test'; import { logContextGetter } from './logContextGetter.js'; import type { OperationRowInsert } from '@nangohq/types'; @@ -54,4 +54,66 @@ describe('model', () => { expect(list3.cursor).toBeNull(); }); }); + describe('messages', () => { + it('should list nothing', async () => { + const list = await listMessages({ limit: 10, parentId: '1' }); + expect(list).toStrictEqual({ + cursorAfter: null, + cursorBefore: null, + count: 0, + items: [] + }); + }); + + it('should paginate', async () => { + const ctx = await logContextGetter.create(operationPayload, { start: false, account, environment }, { logToConsole: false }); + await ctx.info('1'); + await ctx.info('2'); + await ctx.info('3'); + await ctx.info('4'); + + // Should list 2 rows + const list1 = await listMessages({ limit: 2, parentId: ctx.id }); + expect(list1.count).toBe(4); + expect(list1.items).toHaveLength(2); + expect(list1.cursorBefore).toBeDefined(); + expect(list1.cursorAfter).toBeDefined(); + + // After: Should list 2 more rows + const list2 = await listMessages({ limit: 2, parentId: ctx.id, cursorAfter: list1.cursorAfter }); + expect(list2.count).toBe(4); + expect(list2.items).toHaveLength(2); + expect(list2.cursorAfter).toBeDefined(); + expect(list2.items[0]!.id).not.toEqual(list1.items[0]!.id); + + // After: Empty results + // When we get the second operation, it's not possible to know if there are more after so we still need to return a cursor + const list3 = await listMessages({ limit: 1, parentId: ctx.id, cursorAfter: list2.cursorAfter }); + expect(list3.count).toBe(4); + expect(list3.items).toHaveLength(0); + expect(list2.cursorAfter).toBeDefined(); + + // Before: Should list 0 rows before + const list4 = await listMessages({ limit: 2, parentId: ctx.id, cursorBefore: list1.cursorBefore }); + expect(list4.count).toBe(4); + expect(list4.items).toHaveLength(0); + expect(list4.cursorBefore).toBeNull(); + expect(list4.cursorAfter).toBeDefined(); + + // Insert a new row + await ctx.info('4'); + await ctx.info('5'); + + // Before: Should list 1 rows before + const list5 = await listMessages({ limit: 2, parentId: ctx.id, cursorBefore: list1.cursorBefore }); + expect(list5.count).toBe(6); + expect(list5.items).toHaveLength(2); + expect(list5.items[0]?.message).toBe('5'); + expect(list5.items[1]?.message).toBe('4'); + expect(list5.cursorBefore).toBeDefined(); + expect(list5.cursorAfter).toBeDefined(); + expect(list5.items[0]!.id).not.toEqual(list1.items[0]!.id); + expect(list5.cursorBefore).not.toEqual(list1.cursorBefore); + }); + }); }); diff --git a/packages/logs/lib/models/messages.ts b/packages/logs/lib/models/messages.ts index c74a1856a3..12c5df83f6 100644 --- a/packages/logs/lib/models/messages.ts +++ b/packages/logs/lib/models/messages.ts @@ -23,6 +23,8 @@ export interface ListOperations { export interface ListMessages { count: number; items: MessageRow[]; + cursorAfter: string | null; + cursorBefore: string | null; } export interface ListFilters { items: { key: string; doc_count: number }[]; @@ -224,6 +226,8 @@ export async function listMessages(opts: { limit: number; states?: SearchOperationsState[] | undefined; search?: string | undefined; + cursorBefore?: string | null | undefined; + cursorAfter?: string | null | undefined; }): Promise { const query: estypes.QueryDslQueryContainer = { bool: { @@ -248,20 +252,45 @@ export async function listMessages(opts: { }); } + // Sort and cursor + let cursor: any[] | undefined; + let sort: estypes.Sort = [{ createdAt: 'desc' }, { id: 'desc' }]; + if (opts.cursorBefore) { + cursor = parseCursor(opts.cursorBefore); + sort = [{ createdAt: 'asc' }, { id: 'asc' }]; + } else if (opts.cursorAfter) { + cursor = opts.cursorAfter ? parseCursor(opts.cursorAfter) : undefined; + } + const res = await client.search({ index: indexMessages.index, size: opts.limit, - sort: [{ createdAt: 'desc' }, 'id'], + sort, track_total_hits: true, + search_after: cursor, query }); const hits = res.hits; + const total = typeof hits.total === 'object' ? hits.total.value : hits.hits.length; + const items = hits.hits.map((hit) => { + return hit._source!; + }); + if (opts.cursorBefore) { + // In case we set before we have to reverse the message since we inverted the sort + items.reverse(); + return { + count: total, + items, + cursorBefore: hits.hits.length > 0 ? createCursor(hits.hits[hits.hits.length - 1]!) : null, + cursorAfter: null + }; + } return { - count: typeof hits.total === 'object' ? hits.total.value : hits.hits.length, - items: hits.hits.map((hit) => { - return hit._source!; - }) + count: total, + items, + cursorBefore: hits.hits.length > 0 ? createCursor(hits.hits[0]!) : null, + cursorAfter: hits.hits.length > 0 && total > hits.hits.length && hits.hits.length >= opts.limit ? createCursor(hits.hits[hits.hits.length - 1]!) : null }; } diff --git a/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts b/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts index a2a9623a67..0af2a2d2c1 100644 --- a/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts +++ b/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts @@ -81,7 +81,7 @@ describe('POST /logs/messages', () => { expect(res.res.status).toBe(200); expect(res.json).toStrictEqual({ data: [], - pagination: { total: 0 } + pagination: { total: 0, cursor: expect.any(String) } }); }); @@ -137,7 +137,7 @@ describe('POST /logs/messages', () => { userId: null } ], - pagination: { total: 1 } + pagination: { total: 1, cursorBefore: expect.any(String), cursorAfter: expect.any(String) } }); }); diff --git a/packages/server/lib/controllers/v1/logs/searchMessages.ts b/packages/server/lib/controllers/v1/logs/searchMessages.ts index 3e5c457977..98f78e5699 100644 --- a/packages/server/lib/controllers/v1/logs/searchMessages.ts +++ b/packages/server/lib/controllers/v1/logs/searchMessages.ts @@ -7,12 +7,15 @@ import { requireEmptyQuery, zodErrorToHTTP } from '@nangohq/utils'; const validation = z .object({ operationId: operationIdRegex, - limit: z.number().optional().default(100), - search: z.string().optional(), + limit: z.number().max(500).optional().default(100), + search: z.string().max(100).optional(), states: z .array(z.enum(['all', 'waiting', 'running', 'success', 'failed', 'timeout', 'cancelled'])) + .max(10) .optional() - .default(['all']) + .default(['all']), + cursorBefore: z.string().or(z.null()).optional(), + cursorAfter: z.string().or(z.null()).optional() }) .strict(); @@ -59,11 +62,13 @@ export const searchMessages = asyncWrapper(async (req, res) => { parentId: body.operationId, limit: body.limit!, states: body.states, - search: body.search + search: body.search, + cursorBefore: body.cursorBefore, + cursorAfter: body.cursorAfter }); res.status(200).send({ data: rawOps.items, - pagination: { total: rawOps.count } + pagination: { total: rawOps.count, cursorBefore: rawOps.cursorBefore, cursorAfter: rawOps.cursorAfter } }); }); diff --git a/packages/types/lib/logs/api.ts b/packages/types/lib/logs/api.ts index 74957729ac..26dc59509f 100644 --- a/packages/types/lib/logs/api.ts +++ b/packages/types/lib/logs/api.ts @@ -48,10 +48,17 @@ export type SearchMessages = Endpoint<{ Method: 'POST'; Path: '/api/v1/logs/messages'; Querystring: { env: string }; - Body: { operationId: string; limit?: number; states?: SearchOperationsState[]; search?: string | undefined }; + Body: { + operationId: string; + limit?: number; + states?: SearchOperationsState[]; + search?: string | undefined; + cursorBefore?: string | null | undefined; + cursorAfter?: string | null | undefined; + }; Success: { data: MessageRow[]; - pagination: { total: number }; + pagination: { total: number; cursorBefore: string | null; cursorAfter: string | null }; }; }>; export type SearchMessagesData = SearchMessages['Success']['data'][0]; diff --git a/packages/webapp/src/components/ui/Drawer.tsx b/packages/webapp/src/components/ui/Drawer.tsx index 26c9403881..c26a64d57b 100644 --- a/packages/webapp/src/components/ui/Drawer.tsx +++ b/packages/webapp/src/components/ui/Drawer.tsx @@ -24,7 +24,10 @@ const DrawerContent = React.forwardRef {children} diff --git a/packages/webapp/src/components/ui/Table.tsx b/packages/webapp/src/components/ui/Table.tsx index 73fce1d27c..2b150ef420 100644 --- a/packages/webapp/src/components/ui/Table.tsx +++ b/packages/webapp/src/components/ui/Table.tsx @@ -2,9 +2,7 @@ import { forwardRef } from 'react'; import { cn } from '../../utils/utils'; const Table = forwardRef>(({ className, ...props }, ref) => ( -
- - +
)); Table.displayName = 'Table'; diff --git a/packages/webapp/src/hooks/useLogs.tsx b/packages/webapp/src/hooks/useLogs.tsx index 115525650d..bd180b9767 100644 --- a/packages/webapp/src/hooks/useLogs.tsx +++ b/packages/webapp/src/hooks/useLogs.tsx @@ -92,44 +92,59 @@ export function useSearchMessages(env: string, body: SearchMessages['Body']) { const [loading, setLoading] = useState(false); const [data, setData] = useState(); const [error, setError] = useState(); + const signal = useRef(); + + async function manualFetch(opts: Pick) { + if (signal.current && !signal.current.signal.aborted) { + signal.current.abort(); + } - async function fetchData() { setLoading(true); + signal.current = new AbortController(); try { const res = await fetch(`/api/v1/logs/messages?env=${env}`, { method: 'POST', - body: JSON.stringify(body), - headers: { 'Content-Type': 'application/json' } + body: JSON.stringify({ ...body, ...opts }), + headers: { 'Content-Type': 'application/json' }, + signal: signal.current.signal }); if (res.status !== 200) { - setData(undefined); - setError((await res.json()) as SearchMessages['Errors']); - return; + return { error: (await res.json()) as SearchMessages['Errors'] }; } - setError(undefined); - setData((await res.json()) as SearchMessages['Success']); + return { res: (await res.json()) as SearchMessages['Success'] }; } catch (err) { - setData(undefined); - setError(err as any); + if (err instanceof DOMException && err.ABORT_ERR) { + return; + } + return { error: err }; } finally { setLoading(false); } } - useEffect(() => { - if (!loading) { - void fetchData(); + async function fetchData(opts: Pick) { + const man = await manualFetch(opts); + if (!man) { + return; + } + if (man.error) { + setData(undefined); + setError(man.error as any); + return; } - }, [env, body.operationId, body.limit, body.states, body.search]); - function trigger() { + setError(undefined); + setData(man.res); + } + + function trigger(opts: Pick) { if (!loading) { - void fetchData(); + void fetchData(opts); } } - return { data, error, loading, trigger }; + return { data, error, loading, trigger, manualFetch }; } export function useSearchFilters(enabled: boolean, env: string, body: SearchFilters['Body']) { diff --git a/packages/webapp/src/pages/Logs/ShowMessage.tsx b/packages/webapp/src/pages/Logs/ShowMessage.tsx index 458ab771ba..3094f8024c 100644 --- a/packages/webapp/src/pages/Logs/ShowMessage.tsx +++ b/packages/webapp/src/pages/Logs/ShowMessage.tsx @@ -55,7 +55,7 @@ export const ShowMessage: React.FC<{ message: MessageRow }> = ({ message }) => {

Payload

- {message.meta ? ( + {message.meta || message.error ? (
= ({ message }) => { return { code: { padding: '0', whiteSpace: 'pre-wrap' } }; }} > - {JSON.stringify({ error: message.error || undefined, output: message.meta || undefined }, null, 2)} + {JSON.stringify({ error: message.error?.message || undefined, output: message.meta || undefined }, null, 2)}
) : ( diff --git a/packages/webapp/src/pages/Logs/ShowOperation.tsx b/packages/webapp/src/pages/Logs/ShowOperation.tsx index 6fe4062350..2792ef8a60 100644 --- a/packages/webapp/src/pages/Logs/ShowOperation.tsx +++ b/packages/webapp/src/pages/Logs/ShowOperation.tsx @@ -72,7 +72,7 @@ export const ShowOperation: React.FC<{ operationId: string }> = ({ operationId } } return ( -
+

Operation Details

@@ -161,7 +161,7 @@ export const ShowOperation: React.FC<{ operationId: string }> = ({ operationId }
No payload.
)}
- +
); }; diff --git a/packages/webapp/src/pages/Logs/components/MessageRow.tsx b/packages/webapp/src/pages/Logs/components/MessageRow.tsx index e4be7fa584..52f0c037a7 100644 --- a/packages/webapp/src/pages/Logs/components/MessageRow.tsx +++ b/packages/webapp/src/pages/Logs/components/MessageRow.tsx @@ -12,9 +12,11 @@ export const MessageRow: React.FC<{ row: Row }> = ({ row } return ( - + {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + ))} diff --git a/packages/webapp/src/pages/Logs/components/OperationRow.tsx b/packages/webapp/src/pages/Logs/components/OperationRow.tsx index bfd11018f6..5c45b95f21 100644 --- a/packages/webapp/src/pages/Logs/components/OperationRow.tsx +++ b/packages/webapp/src/pages/Logs/components/OperationRow.tsx @@ -19,7 +19,7 @@ export const OperationRow: React.FC<{ row: Row }> = ({ row -
+
diff --git a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx index 9e53a23ffd..9b87517133 100644 --- a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx +++ b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx @@ -2,7 +2,7 @@ import type { ColumnDef } from '@tanstack/react-table'; import { flexRender, getCoreRowModel, useReactTable } from '@tanstack/react-table'; import { Input } from '../../../components/ui/input/Input'; import { useSearchMessages } from '../../../hooks/useLogs'; -import type { SearchOperationsData } from '@nangohq/types'; +import type { SearchMessages, SearchOperationsData } from '@nangohq/types'; import { formatDateToLogFormat, formatQuantity } from '../../../utils/utils'; import { useStore } from '../../../store'; import * as Table from '../../../components/ui/Table'; @@ -11,10 +11,11 @@ import Info from '../../../components/ui/Info'; import { LevelTag } from './LevelTag'; import { MessageRow } from './MessageRow'; import { ChevronRightIcon, MagnifyingGlassIcon } from '@radix-ui/react-icons'; -import { useMemo, useState } from 'react'; -import { useDebounce, useInterval } from 'react-use'; +import { useEffect, useMemo, useRef, useState } from 'react'; +import { useDebounce, useIntersection, useInterval } from 'react-use'; import { Tag } from './Tag'; import { Skeleton } from '../../../components/ui/Skeleton'; +import Button from '../../../components/ui/button/Button'; export const columns: ColumnDef[] = [ { @@ -63,35 +64,121 @@ export const columns: ColumnDef[] = [ } ]; -export const SearchInOperation: React.FC<{ operationId: string; live: boolean }> = ({ operationId, live }) => { +const limit = 50; + +export const SearchInOperation: React.FC<{ operationId: string; isLive: boolean }> = ({ operationId, isLive }) => { const env = useStore((state) => state.env); + // --- Data fetch const [search, setSearch] = useState(); const [debouncedSearch, setDebouncedSearch] = useState(); - const { data, error, loading, trigger } = useSearchMessages(env, { limit: 20, operationId, search: debouncedSearch }); + const cursorBefore = useRef(); + const cursorAfter = useRef(); + const [hasLoadedMore, setHasLoadedMore] = useState(false); + const [readyToDisplay, setReadyToDisplay] = useState(false); + const { data, error, loading, trigger, manualFetch } = useSearchMessages(env, { limit, operationId, search: debouncedSearch }); + const [messages, setMessages] = useState([]); + + useDebounce(() => setDebouncedSearch(search), 250, [search]); + useEffect(() => { + // Data aggregation to enable infinite scroll + // Because states are changing we need to deduplicate and update rows + setMessages((prev) => { + if (prev.length <= 0 || !data?.data) { + return data?.data || []; + } + + const next = data.data; + for (const item of prev) { + if (next.find((n) => n.id === item.id)) { + continue; + } + next.push(item); + } + + return next; + }); + if (data?.pagination.cursorBefore) { + cursorBefore.current = data?.pagination.cursorBefore; + } + setReadyToDisplay(true); + }, [data?.data]); + useEffect(() => { + if (data?.pagination.cursorAfter && !hasLoadedMore) { + // We set the cursor only on first page (if we haven't hit a next page) + // Otherwise the live refresh will erase + cursorAfter.current = data.pagination.cursorAfter; + } + }, [hasLoadedMore, data]); + useDebounce( + () => { + // We clear the cursor because it's a brand new search + cursorAfter.current = null; + // Debounce the trigger to avoid spamming the backend and avoid conflict with rapid filter change + trigger({}); + }, + 200, + [] + ); + // --- Table Display const table = useReactTable({ - data: data ? data.data : [], + data: messages, columns, getCoreRowModel: getCoreRowModel() }); - useDebounce(() => setDebouncedSearch(search), 250, [search]); - - const total = useMemo(() => { + const totalHumanReadable = useMemo(() => { if (!data?.pagination) { return 0; } return formatQuantity(data.pagination.total); }, [data?.pagination]); + // --- Live // auto refresh useInterval( - () => { - // Auto refresh - trigger(); + function onAutoRefresh() { + trigger({ cursorBefore: cursorBefore.current }); }, - live ? 5000 : null + isLive && !loading ? 5000 : 5000 ); + // --- Infinite scroll + // We use the cursor manually because we want to keep refreshing the head even we add stuff to the tail + const bottomScrollRef = useRef(null); + const bottomScroll = useIntersection(bottomScrollRef, { + root: null, + rootMargin: '0px', + threshold: 1 + }); + const appendItems = async () => { + if (!cursorAfter.current) { + return; + } + const rows = await manualFetch({ cursorAfter: cursorAfter.current }); + if (!rows || 'error' in rows) { + return; + } + + cursorAfter.current = rows.res.pagination.cursorAfter; + setHasLoadedMore(true); + setMessages((prev) => [...prev, ...rows.res.data]); + }; + useEffect(() => { + // when the load more button is fully in view + if (!bottomScroll || !bottomScroll.isIntersecting) { + return; + } + if (cursorAfter.current && !loading) { + void appendItems(); + } + }, [bottomScroll, loading, bottomScrollRef]); + + const loadMore = () => { + if (!loading) { + void appendItems(); + } + }; + if (!data && loading) { return (
@@ -104,10 +191,10 @@ export const SearchInOperation: React.FC<{ operationId: string; live: boolean }> } return ( -
+

Logs {loading && }

-
{total} logs found
+
{totalHumanReadable} logs found
onChange={(e) => setSearch(e.target.value)} />
-
+
{error && ( An error occurred )} - - + + {table.getHeaderGroups().map((headerGroup) => ( {headerGroup.headers.map((header) => { @@ -143,15 +230,33 @@ export const SearchInOperation: React.FC<{ operationId: string; live: boolean }> ))} - + {table.getRowModel().rows?.length ? ( table.getRowModel().rows.map((row) => ) - ) : ( + ) : messages.length <= 0 && !loading && readyToDisplay ? ( No results. + ) : ( + + {table.getAllColumns().map((col, i) => { + return ( + + + + ); + })} + + )} + + {data && data.pagination.total > 0 && data.data.length > 0 && data.pagination && cursorAfter.current && readyToDisplay && ( +
+ +
)} From ea09b22f9e8c0b53d1ad29ce216f40dc813f96c0 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 28 May 2024 10:48:18 +0200 Subject: [PATCH 5/9] refresh rate --- packages/webapp/src/pages/Logs/components/SearchInOperation.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx index 9b87517133..bfcf282157 100644 --- a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx +++ b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx @@ -139,7 +139,7 @@ export const SearchInOperation: React.FC<{ operationId: string; isLive: boolean function onAutoRefresh() { trigger({ cursorBefore: cursorBefore.current }); }, - isLive && !loading ? 5000 : 5000 + isLive && !loading ? 5000 : null ); // --- Infinite scroll From 7c2f317e321710890b0b9dc38e0cb6bc4f25fea6 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Wed, 29 May 2024 11:01:54 +0200 Subject: [PATCH 6/9] review --- packages/logs/lib/models/messages.ts | 3 ++- packages/webapp/src/pages/Logs/Search.tsx | 10 ++++++++-- packages/webapp/src/pages/Logs/ShowMessage.tsx | 1 + .../webapp/src/pages/Logs/components/MessageRow.tsx | 6 +++--- .../webapp/src/pages/Logs/components/OperationRow.tsx | 2 +- .../src/pages/Logs/components/SearchInOperation.tsx | 2 +- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/logs/lib/models/messages.ts b/packages/logs/lib/models/messages.ts index c74a1856a3..e98eb96a9b 100644 --- a/packages/logs/lib/models/messages.ts +++ b/packages/logs/lib/models/messages.ts @@ -144,12 +144,13 @@ export async function listOperations(opts: { const hits = res.hits; const total = typeof hits.total === 'object' ? hits.total.value : hits.hits.length; + const totalPage = hits.hits.length; return { count: total, items: hits.hits.map((hit) => { return hit._source!; }), - cursor: hits.hits.length > 0 && total > hits.hits.length && hits.hits.length >= opts.limit ? createCursor(hits.hits[hits.hits.length - 1]!) : null + cursor: totalPage > 0 && total > totalPage && opts.limit >= totalPage ? createCursor(hits.hits[hits.hits.length - 1]!) : null }; } diff --git a/packages/webapp/src/pages/Logs/Search.tsx b/packages/webapp/src/pages/Logs/Search.tsx index b2de0cee34..475de6ea73 100644 --- a/packages/webapp/src/pages/Logs/Search.tsx +++ b/packages/webapp/src/pages/Logs/Search.tsx @@ -302,7 +302,7 @@ export const LogsSearch: React.FC = () => { {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ) + table.getRowModel().rows.map((row) => ) ) : operations.length <= 0 && !loading && readyToDisplay ? ( @@ -325,7 +325,13 @@ export const LogsSearch: React.FC = () => { {data && data.pagination.total > 0 && data.data.length > 0 && cursor.current && readyToDisplay && (
)} diff --git a/packages/webapp/src/pages/Logs/ShowMessage.tsx b/packages/webapp/src/pages/Logs/ShowMessage.tsx index 458ab771ba..1ef2e5fb73 100644 --- a/packages/webapp/src/pages/Logs/ShowMessage.tsx +++ b/packages/webapp/src/pages/Logs/ShowMessage.tsx @@ -47,6 +47,7 @@ export const ShowMessage: React.FC<{ message: MessageRow }> = ({ message }) => { styles={() => { return { code: { padding: '0', whiteSpace: 'pre-wrap' } }; }} + noCopy > {message.message} diff --git a/packages/webapp/src/pages/Logs/components/MessageRow.tsx b/packages/webapp/src/pages/Logs/components/MessageRow.tsx index e4be7fa584..42b25528f2 100644 --- a/packages/webapp/src/pages/Logs/components/MessageRow.tsx +++ b/packages/webapp/src/pages/Logs/components/MessageRow.tsx @@ -10,7 +10,7 @@ import { ArrowLeftIcon } from '@radix-ui/react-icons'; const drawerWidth = '834px'; export const MessageRow: React.FC<{ row: Row }> = ({ row }) => { return ( - + {row.getVisibleCells().map((cell) => ( @@ -19,8 +19,8 @@ export const MessageRow: React.FC<{ row: Row }> = ({ row } -
-
+
+
diff --git a/packages/webapp/src/pages/Logs/components/OperationRow.tsx b/packages/webapp/src/pages/Logs/components/OperationRow.tsx index bfd11018f6..8f375c3e19 100644 --- a/packages/webapp/src/pages/Logs/components/OperationRow.tsx +++ b/packages/webapp/src/pages/Logs/components/OperationRow.tsx @@ -19,7 +19,7 @@ export const OperationRow: React.FC<{ row: Row }> = ({ row -
+
diff --git a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx index 9e53a23ffd..9eb93266d9 100644 --- a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx +++ b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx @@ -145,7 +145,7 @@ export const SearchInOperation: React.FC<{ operationId: string; live: boolean }> {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ) + table.getRowModel().rows.map((row) => ) ) : ( From 91da4c66b67a2eb47c9d48b068614ba7be9dddaa Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Wed, 29 May 2024 13:53:05 +0200 Subject: [PATCH 7/9] test --- packages/logs/lib/models/messages.ts | 7 ++++--- .../controllers/v1/logs/searchMessages.integration.test.ts | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/logs/lib/models/messages.ts b/packages/logs/lib/models/messages.ts index 15e5f2c593..27d904381a 100644 --- a/packages/logs/lib/models/messages.ts +++ b/packages/logs/lib/models/messages.ts @@ -274,6 +274,7 @@ export async function listMessages(opts: { const hits = res.hits; const total = typeof hits.total === 'object' ? hits.total.value : hits.hits.length; + const totalPage = hits.hits.length; const items = hits.hits.map((hit) => { return hit._source!; }); @@ -283,15 +284,15 @@ export async function listMessages(opts: { return { count: total, items, - cursorBefore: hits.hits.length > 0 ? createCursor(hits.hits[hits.hits.length - 1]!) : null, + cursorBefore: totalPage > 0 ? createCursor(hits.hits[hits.hits.length - 1]!) : null, cursorAfter: null }; } return { count: total, items, - cursorBefore: hits.hits.length > 0 ? createCursor(hits.hits[0]!) : null, - cursorAfter: hits.hits.length > 0 && total > hits.hits.length && hits.hits.length >= opts.limit ? createCursor(hits.hits[hits.hits.length - 1]!) : null + cursorBefore: totalPage > 0 ? createCursor(hits.hits[0]!) : null, + cursorAfter: totalPage > 0 && total > totalPage && hits.hits.length >= opts.limit ? createCursor(hits.hits[hits.hits.length - 1]!) : null }; } diff --git a/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts b/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts index 0af2a2d2c1..89d35068c9 100644 --- a/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts +++ b/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts @@ -79,9 +79,9 @@ describe('POST /logs/messages', () => { isSuccess(res.json); expect(res.res.status).toBe(200); - expect(res.json).toStrictEqual({ + expect(res.json).toStrictEqual({ data: [], - pagination: { total: 0, cursor: expect.any(String) } + pagination: { total: 0, cursorAfter: null, cursorBefore: null } }); }); @@ -137,7 +137,7 @@ describe('POST /logs/messages', () => { userId: null } ], - pagination: { total: 1, cursorBefore: expect.any(String), cursorAfter: expect.any(String) } + pagination: { total: 1, cursorBefore: expect.any(String), cursorAfter: null } }); }); From b9610c8af516d483f371fa379daeb8ff273d786b Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Wed, 29 May 2024 14:08:52 +0200 Subject: [PATCH 8/9] load more --- .../src/pages/Logs/components/SearchInOperation.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx index 06894c5692..e66eb86a60 100644 --- a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx +++ b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx @@ -254,7 +254,13 @@ export const SearchInOperation: React.FC<{ operationId: string; isLive: boolean {data && data.pagination.total > 0 && data.data.length > 0 && data.pagination && cursorAfter.current && readyToDisplay && (
)} From 9c1e6142e753cfe0002906ce0c0403285c4d6a42 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Wed, 29 May 2024 15:04:26 +0200 Subject: [PATCH 9/9] review --- packages/logs/lib/models/messages.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/logs/lib/models/messages.ts b/packages/logs/lib/models/messages.ts index d971441b39..5d2722fbb5 100644 --- a/packages/logs/lib/models/messages.ts +++ b/packages/logs/lib/models/messages.ts @@ -257,6 +257,8 @@ export async function listMessages(opts: { let cursor: any[] | undefined; let sort: estypes.Sort = [{ createdAt: 'desc' }, { id: 'desc' }]; if (opts.cursorBefore) { + // search_before does not exists so we reverse the sort + // https://github.com/elastic/elasticsearch/issues/29449 cursor = parseCursor(opts.cursorBefore); sort = [{ createdAt: 'asc' }, { id: 'asc' }]; } else if (opts.cursorAfter) { @@ -278,6 +280,7 @@ export async function listMessages(opts: { const items = hits.hits.map((hit) => { return hit._source!; }); + if (opts.cursorBefore) { // In case we set before we have to reverse the message since we inverted the sort items.reverse(); @@ -288,6 +291,7 @@ export async function listMessages(opts: { cursorAfter: null }; } + return { count: total, items,