diff --git a/packages/logs/lib/models/messages.integration.test.ts b/packages/logs/lib/models/messages.integration.test.ts index a9e616d495..e6d49294e6 100644 --- a/packages/logs/lib/models/messages.integration.test.ts +++ b/packages/logs/lib/models/messages.integration.test.ts @@ -1,7 +1,7 @@ import { describe, beforeAll, it, expect, vi } from 'vitest'; import { deleteIndex, migrateMapping } from '../es/helpers.js'; -import type { ListOperations } from './messages.js'; -import { listOperations } from './messages.js'; +import type { ListMessages, ListOperations } from './messages.js'; +import { listMessages, listOperations } from './messages.js'; import { afterEach } from 'node:test'; import { logContextGetter } from './logContextGetter.js'; import type { OperationRowInsert } from '@nangohq/types'; @@ -54,4 +54,67 @@ describe('model', () => { expect(list3.cursor).toBeNull(); }); }); + + describe('messages', () => { + it('should list nothing', async () => { + const list = await listMessages({ limit: 10, parentId: '1' }); + expect(list).toStrictEqual({ + cursorAfter: null, + cursorBefore: null, + count: 0, + items: [] + }); + }); + + it('should paginate', async () => { + const ctx = await logContextGetter.create(operationPayload, { start: false, account, environment }, { logToConsole: false }); + await ctx.info('1'); + await ctx.info('2'); + await ctx.info('3'); + await ctx.info('4'); + + // Should list 2 rows + const list1 = await listMessages({ limit: 2, parentId: ctx.id }); + expect(list1.count).toBe(4); + expect(list1.items).toHaveLength(2); + expect(list1.cursorBefore).toBeDefined(); + expect(list1.cursorAfter).toBeDefined(); + + // After: Should list 2 more rows + const list2 = await listMessages({ limit: 2, parentId: ctx.id, cursorAfter: list1.cursorAfter }); + expect(list2.count).toBe(4); + expect(list2.items).toHaveLength(2); + expect(list2.cursorAfter).toBeDefined(); + expect(list2.items[0]!.id).not.toEqual(list1.items[0]!.id); + + // After: Empty results + // When we get the second operation, it's not possible to know if there are more after so we still need to return a cursor + const list3 = await listMessages({ limit: 1, parentId: ctx.id, cursorAfter: list2.cursorAfter }); + expect(list3.count).toBe(4); + expect(list3.items).toHaveLength(0); + expect(list2.cursorAfter).toBeDefined(); + + // Before: Should list 0 rows before + const list4 = await listMessages({ limit: 2, parentId: ctx.id, cursorBefore: list1.cursorBefore }); + expect(list4.count).toBe(4); + expect(list4.items).toHaveLength(0); + expect(list4.cursorBefore).toBeNull(); + expect(list4.cursorAfter).toBeDefined(); + + // Insert a new row + await ctx.info('4'); + await ctx.info('5'); + + // Before: Should list 1 rows before + const list5 = await listMessages({ limit: 2, parentId: ctx.id, cursorBefore: list1.cursorBefore }); + expect(list5.count).toBe(6); + expect(list5.items).toHaveLength(2); + expect(list5.items[0]?.message).toBe('5'); + expect(list5.items[1]?.message).toBe('4'); + expect(list5.cursorBefore).toBeDefined(); + expect(list5.cursorAfter).toBeDefined(); + expect(list5.items[0]!.id).not.toEqual(list1.items[0]!.id); + expect(list5.cursorBefore).not.toEqual(list1.cursorBefore); + }); + }); }); diff --git a/packages/logs/lib/models/messages.ts b/packages/logs/lib/models/messages.ts index c76e2a0342..5d2722fbb5 100644 --- a/packages/logs/lib/models/messages.ts +++ b/packages/logs/lib/models/messages.ts @@ -23,6 +23,8 @@ export interface ListOperations { export interface ListMessages { count: number; items: MessageRow[]; + cursorAfter: string | null; + cursorBefore: string | null; } export interface ListFilters { items: { key: string; doc_count: number }[]; @@ -225,6 +227,8 @@ export async function listMessages(opts: { limit: number; states?: SearchOperationsState[] | undefined; search?: string | undefined; + cursorBefore?: string | null | undefined; + cursorAfter?: string | null | undefined; }): Promise { const query: estypes.QueryDslQueryContainer = { bool: { @@ -249,20 +253,50 @@ export async function listMessages(opts: { }); } + // Sort and cursor + let cursor: any[] | undefined; + let sort: estypes.Sort = [{ createdAt: 'desc' }, { id: 'desc' }]; + if (opts.cursorBefore) { + // search_before does not exists so we reverse the sort + // https://github.com/elastic/elasticsearch/issues/29449 + cursor = parseCursor(opts.cursorBefore); + sort = [{ createdAt: 'asc' }, { id: 'asc' }]; + } else if (opts.cursorAfter) { + cursor = opts.cursorAfter ? parseCursor(opts.cursorAfter) : undefined; + } + const res = await client.search({ index: indexMessages.index, size: opts.limit, - sort: [{ createdAt: 'desc' }, 'id'], + sort, track_total_hits: true, + search_after: cursor, query }); const hits = res.hits; + const total = typeof hits.total === 'object' ? hits.total.value : hits.hits.length; + const totalPage = hits.hits.length; + const items = hits.hits.map((hit) => { + return hit._source!; + }); + + if (opts.cursorBefore) { + // In case we set before we have to reverse the message since we inverted the sort + items.reverse(); + return { + count: total, + items, + cursorBefore: totalPage > 0 ? createCursor(hits.hits[hits.hits.length - 1]!) : null, + cursorAfter: null + }; + } + return { - count: typeof hits.total === 'object' ? hits.total.value : hits.hits.length, - items: hits.hits.map((hit) => { - return hit._source!; - }) + count: total, + items, + cursorBefore: totalPage > 0 ? createCursor(hits.hits[0]!) : null, + cursorAfter: totalPage > 0 && total > totalPage && totalPage >= opts.limit ? createCursor(hits.hits[hits.hits.length - 1]!) : null }; } diff --git a/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts b/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts index a2a9623a67..89d35068c9 100644 --- a/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts +++ b/packages/server/lib/controllers/v1/logs/searchMessages.integration.test.ts @@ -79,9 +79,9 @@ describe('POST /logs/messages', () => { isSuccess(res.json); expect(res.res.status).toBe(200); - expect(res.json).toStrictEqual({ + expect(res.json).toStrictEqual({ data: [], - pagination: { total: 0 } + pagination: { total: 0, cursorAfter: null, cursorBefore: null } }); }); @@ -137,7 +137,7 @@ describe('POST /logs/messages', () => { userId: null } ], - pagination: { total: 1 } + pagination: { total: 1, cursorBefore: expect.any(String), cursorAfter: null } }); }); diff --git a/packages/server/lib/controllers/v1/logs/searchMessages.ts b/packages/server/lib/controllers/v1/logs/searchMessages.ts index 3e5c457977..98f78e5699 100644 --- a/packages/server/lib/controllers/v1/logs/searchMessages.ts +++ b/packages/server/lib/controllers/v1/logs/searchMessages.ts @@ -7,12 +7,15 @@ import { requireEmptyQuery, zodErrorToHTTP } from '@nangohq/utils'; const validation = z .object({ operationId: operationIdRegex, - limit: z.number().optional().default(100), - search: z.string().optional(), + limit: z.number().max(500).optional().default(100), + search: z.string().max(100).optional(), states: z .array(z.enum(['all', 'waiting', 'running', 'success', 'failed', 'timeout', 'cancelled'])) + .max(10) .optional() - .default(['all']) + .default(['all']), + cursorBefore: z.string().or(z.null()).optional(), + cursorAfter: z.string().or(z.null()).optional() }) .strict(); @@ -59,11 +62,13 @@ export const searchMessages = asyncWrapper(async (req, res) => { parentId: body.operationId, limit: body.limit!, states: body.states, - search: body.search + search: body.search, + cursorBefore: body.cursorBefore, + cursorAfter: body.cursorAfter }); res.status(200).send({ data: rawOps.items, - pagination: { total: rawOps.count } + pagination: { total: rawOps.count, cursorBefore: rawOps.cursorBefore, cursorAfter: rawOps.cursorAfter } }); }); diff --git a/packages/types/lib/logs/api.ts b/packages/types/lib/logs/api.ts index 74957729ac..26dc59509f 100644 --- a/packages/types/lib/logs/api.ts +++ b/packages/types/lib/logs/api.ts @@ -48,10 +48,17 @@ export type SearchMessages = Endpoint<{ Method: 'POST'; Path: '/api/v1/logs/messages'; Querystring: { env: string }; - Body: { operationId: string; limit?: number; states?: SearchOperationsState[]; search?: string | undefined }; + Body: { + operationId: string; + limit?: number; + states?: SearchOperationsState[]; + search?: string | undefined; + cursorBefore?: string | null | undefined; + cursorAfter?: string | null | undefined; + }; Success: { data: MessageRow[]; - pagination: { total: number }; + pagination: { total: number; cursorBefore: string | null; cursorAfter: string | null }; }; }>; export type SearchMessagesData = SearchMessages['Success']['data'][0]; diff --git a/packages/webapp/src/components/ui/Drawer.tsx b/packages/webapp/src/components/ui/Drawer.tsx index 26c9403881..c26a64d57b 100644 --- a/packages/webapp/src/components/ui/Drawer.tsx +++ b/packages/webapp/src/components/ui/Drawer.tsx @@ -24,7 +24,10 @@ const DrawerContent = React.forwardRef {children} diff --git a/packages/webapp/src/components/ui/Table.tsx b/packages/webapp/src/components/ui/Table.tsx index 73fce1d27c..2b150ef420 100644 --- a/packages/webapp/src/components/ui/Table.tsx +++ b/packages/webapp/src/components/ui/Table.tsx @@ -2,9 +2,7 @@ import { forwardRef } from 'react'; import { cn } from '../../utils/utils'; const Table = forwardRef>(({ className, ...props }, ref) => ( -
- - +
)); Table.displayName = 'Table'; diff --git a/packages/webapp/src/hooks/useLogs.tsx b/packages/webapp/src/hooks/useLogs.tsx index 115525650d..bd180b9767 100644 --- a/packages/webapp/src/hooks/useLogs.tsx +++ b/packages/webapp/src/hooks/useLogs.tsx @@ -92,44 +92,59 @@ export function useSearchMessages(env: string, body: SearchMessages['Body']) { const [loading, setLoading] = useState(false); const [data, setData] = useState(); const [error, setError] = useState(); + const signal = useRef(); + + async function manualFetch(opts: Pick) { + if (signal.current && !signal.current.signal.aborted) { + signal.current.abort(); + } - async function fetchData() { setLoading(true); + signal.current = new AbortController(); try { const res = await fetch(`/api/v1/logs/messages?env=${env}`, { method: 'POST', - body: JSON.stringify(body), - headers: { 'Content-Type': 'application/json' } + body: JSON.stringify({ ...body, ...opts }), + headers: { 'Content-Type': 'application/json' }, + signal: signal.current.signal }); if (res.status !== 200) { - setData(undefined); - setError((await res.json()) as SearchMessages['Errors']); - return; + return { error: (await res.json()) as SearchMessages['Errors'] }; } - setError(undefined); - setData((await res.json()) as SearchMessages['Success']); + return { res: (await res.json()) as SearchMessages['Success'] }; } catch (err) { - setData(undefined); - setError(err as any); + if (err instanceof DOMException && err.ABORT_ERR) { + return; + } + return { error: err }; } finally { setLoading(false); } } - useEffect(() => { - if (!loading) { - void fetchData(); + async function fetchData(opts: Pick) { + const man = await manualFetch(opts); + if (!man) { + return; + } + if (man.error) { + setData(undefined); + setError(man.error as any); + return; } - }, [env, body.operationId, body.limit, body.states, body.search]); - function trigger() { + setError(undefined); + setData(man.res); + } + + function trigger(opts: Pick) { if (!loading) { - void fetchData(); + void fetchData(opts); } } - return { data, error, loading, trigger }; + return { data, error, loading, trigger, manualFetch }; } export function useSearchFilters(enabled: boolean, env: string, body: SearchFilters['Body']) { diff --git a/packages/webapp/src/pages/Logs/ShowMessage.tsx b/packages/webapp/src/pages/Logs/ShowMessage.tsx index 1ef2e5fb73..0dd8843781 100644 --- a/packages/webapp/src/pages/Logs/ShowMessage.tsx +++ b/packages/webapp/src/pages/Logs/ShowMessage.tsx @@ -56,7 +56,7 @@ export const ShowMessage: React.FC<{ message: MessageRow }> = ({ message }) => {

Payload

- {message.meta ? ( + {message.meta || message.error ? (
= ({ message }) => { return { code: { padding: '0', whiteSpace: 'pre-wrap' } }; }} > - {JSON.stringify({ error: message.error || undefined, output: message.meta || undefined }, null, 2)} + {JSON.stringify({ error: message.error?.message || undefined, output: message.meta || undefined }, null, 2)}
) : ( diff --git a/packages/webapp/src/pages/Logs/ShowOperation.tsx b/packages/webapp/src/pages/Logs/ShowOperation.tsx index 6fe4062350..2792ef8a60 100644 --- a/packages/webapp/src/pages/Logs/ShowOperation.tsx +++ b/packages/webapp/src/pages/Logs/ShowOperation.tsx @@ -72,7 +72,7 @@ export const ShowOperation: React.FC<{ operationId: string }> = ({ operationId } } return ( -
+

Operation Details

@@ -161,7 +161,7 @@ export const ShowOperation: React.FC<{ operationId: string }> = ({ operationId }
No payload.
)}
- +
); }; diff --git a/packages/webapp/src/pages/Logs/components/MessageRow.tsx b/packages/webapp/src/pages/Logs/components/MessageRow.tsx index 42b25528f2..7009ecd49f 100644 --- a/packages/webapp/src/pages/Logs/components/MessageRow.tsx +++ b/packages/webapp/src/pages/Logs/components/MessageRow.tsx @@ -12,9 +12,11 @@ export const MessageRow: React.FC<{ row: Row }> = ({ row } return ( - + {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + ))} diff --git a/packages/webapp/src/pages/Logs/components/OperationRow.tsx b/packages/webapp/src/pages/Logs/components/OperationRow.tsx index 8f375c3e19..a89ea5d68b 100644 --- a/packages/webapp/src/pages/Logs/components/OperationRow.tsx +++ b/packages/webapp/src/pages/Logs/components/OperationRow.tsx @@ -19,7 +19,7 @@ export const OperationRow: React.FC<{ row: Row }> = ({ row -
+
diff --git a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx index 9eb93266d9..e66eb86a60 100644 --- a/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx +++ b/packages/webapp/src/pages/Logs/components/SearchInOperation.tsx @@ -2,7 +2,7 @@ import type { ColumnDef } from '@tanstack/react-table'; import { flexRender, getCoreRowModel, useReactTable } from '@tanstack/react-table'; import { Input } from '../../../components/ui/input/Input'; import { useSearchMessages } from '../../../hooks/useLogs'; -import type { SearchOperationsData } from '@nangohq/types'; +import type { SearchMessages, SearchOperationsData } from '@nangohq/types'; import { formatDateToLogFormat, formatQuantity } from '../../../utils/utils'; import { useStore } from '../../../store'; import * as Table from '../../../components/ui/Table'; @@ -11,10 +11,11 @@ import Info from '../../../components/ui/Info'; import { LevelTag } from './LevelTag'; import { MessageRow } from './MessageRow'; import { ChevronRightIcon, MagnifyingGlassIcon } from '@radix-ui/react-icons'; -import { useMemo, useState } from 'react'; -import { useDebounce, useInterval } from 'react-use'; +import { useEffect, useMemo, useRef, useState } from 'react'; +import { useDebounce, useIntersection, useInterval } from 'react-use'; import { Tag } from './Tag'; import { Skeleton } from '../../../components/ui/Skeleton'; +import Button from '../../../components/ui/button/Button'; export const columns: ColumnDef[] = [ { @@ -63,35 +64,121 @@ export const columns: ColumnDef[] = [ } ]; -export const SearchInOperation: React.FC<{ operationId: string; live: boolean }> = ({ operationId, live }) => { +const limit = 50; + +export const SearchInOperation: React.FC<{ operationId: string; isLive: boolean }> = ({ operationId, isLive }) => { const env = useStore((state) => state.env); + // --- Data fetch const [search, setSearch] = useState(); const [debouncedSearch, setDebouncedSearch] = useState(); - const { data, error, loading, trigger } = useSearchMessages(env, { limit: 20, operationId, search: debouncedSearch }); + const cursorBefore = useRef(); + const cursorAfter = useRef(); + const [hasLoadedMore, setHasLoadedMore] = useState(false); + const [readyToDisplay, setReadyToDisplay] = useState(false); + const { data, error, loading, trigger, manualFetch } = useSearchMessages(env, { limit, operationId, search: debouncedSearch }); + const [messages, setMessages] = useState([]); + + useDebounce(() => setDebouncedSearch(search), 250, [search]); + useEffect(() => { + // Data aggregation to enable infinite scroll + // Because states are changing we need to deduplicate and update rows + setMessages((prev) => { + if (prev.length <= 0 || !data?.data) { + return data?.data || []; + } + + const next = data.data; + for (const item of prev) { + if (next.find((n) => n.id === item.id)) { + continue; + } + next.push(item); + } + + return next; + }); + if (data?.pagination.cursorBefore) { + cursorBefore.current = data?.pagination.cursorBefore; + } + setReadyToDisplay(true); + }, [data?.data]); + useEffect(() => { + if (data?.pagination.cursorAfter && !hasLoadedMore) { + // We set the cursor only on first page (if we haven't hit a next page) + // Otherwise the live refresh will erase + cursorAfter.current = data.pagination.cursorAfter; + } + }, [hasLoadedMore, data]); + useDebounce( + () => { + // We clear the cursor because it's a brand new search + cursorAfter.current = null; + // Debounce the trigger to avoid spamming the backend and avoid conflict with rapid filter change + trigger({}); + }, + 200, + [] + ); + // --- Table Display const table = useReactTable({ - data: data ? data.data : [], + data: messages, columns, getCoreRowModel: getCoreRowModel() }); - useDebounce(() => setDebouncedSearch(search), 250, [search]); - - const total = useMemo(() => { + const totalHumanReadable = useMemo(() => { if (!data?.pagination) { return 0; } return formatQuantity(data.pagination.total); }, [data?.pagination]); + // --- Live // auto refresh useInterval( - () => { - // Auto refresh - trigger(); + function onAutoRefresh() { + trigger({ cursorBefore: cursorBefore.current }); }, - live ? 5000 : null + isLive && !loading ? 5000 : null ); + // --- Infinite scroll + // We use the cursor manually because we want to keep refreshing the head even we add stuff to the tail + const bottomScrollRef = useRef(null); + const bottomScroll = useIntersection(bottomScrollRef, { + root: null, + rootMargin: '0px', + threshold: 1 + }); + const appendItems = async () => { + if (!cursorAfter.current) { + return; + } + const rows = await manualFetch({ cursorAfter: cursorAfter.current }); + if (!rows || 'error' in rows) { + return; + } + + cursorAfter.current = rows.res.pagination.cursorAfter; + setHasLoadedMore(true); + setMessages((prev) => [...prev, ...rows.res.data]); + }; + useEffect(() => { + // when the load more button is fully in view + if (!bottomScroll || !bottomScroll.isIntersecting) { + return; + } + if (cursorAfter.current && !loading) { + void appendItems(); + } + }, [bottomScroll, loading, bottomScrollRef]); + + const loadMore = () => { + if (!loading) { + void appendItems(); + } + }; + if (!data && loading) { return (
@@ -104,10 +191,10 @@ export const SearchInOperation: React.FC<{ operationId: string; live: boolean }> } return ( -
+

Logs {loading && }

-
{total} logs found
+
{totalHumanReadable} logs found
onChange={(e) => setSearch(e.target.value)} />
-
+
{error && ( An error occurred )} - - + + {table.getHeaderGroups().map((headerGroup) => ( {headerGroup.headers.map((header) => { @@ -143,15 +230,39 @@ export const SearchInOperation: React.FC<{ operationId: string; live: boolean }> ))} - + {table.getRowModel().rows?.length ? ( table.getRowModel().rows.map((row) => ) - ) : ( + ) : messages.length <= 0 && !loading && readyToDisplay ? ( No results. + ) : ( + + {table.getAllColumns().map((col, i) => { + return ( + + + + ); + })} + + )} + + {data && data.pagination.total > 0 && data.data.length > 0 && data.pagination && cursorAfter.current && readyToDisplay && ( +
+ +
)}