Improve the comprehensive pagination engine

- Add documentation
- Rename things for clarity
- Add support for transforming the data, besides filtering
- Add support for also specifying filters as an array
- Add a bunch of more status fields

.changelog/1385.internal.md

@@ -0,0 +1 @@
+Improve the comprehensive pagination engine

src/app/components/Table/PaginationEngine.ts

@@ -7,15 +7,95 @@ export interface SimplePaginationEngine {
   linkToPage: (pageNumber: number) => To
 }
 
-export interface PaginatedResults<Item> {
+/**
+ * The data returned by a comprehensive pagination engine to the data consumer component
+ */
+export interface ComprehensivePaginatedResults<Item, ExtractedData = typeof undefined> {
+  /**
+   * Control interface that can be plugged to a Table's `pagination` prop
+   */
   tablePaginationProps: TablePaginationProps
+
+  /**
+   * The data provided to the data consumer in the current window
+   */
   data: Item[] | undefined
+
+  /**
+   * Any extra data produced by the transformer function (besides the array of items)
+   */
+  extractedData?: ExtractedData | undefined
+
+  /**
+   * Is the data set still loading from the server?
+   */
+  isLoading: boolean
+
+  /**
+   * Has the data been loaded from the server?
+   */
+  isFetched: boolean
+
+  /**
+   * Are we on the first page of the pagination?
+   */
+  isOnFirstPage: boolean
+
+  /**
+   * Do we have any data on the client page?
+   */
+  hasData: boolean
+
+  /**
+   * Can we say that there are no results at all?
+   *
+   * This is determined before any filtering or transformation.
+   */
+  hasNoResultsWhatsoever: boolean
+
+  /**
+   * Can we say that there are no results on the selected page
+   *
+   * This will only be marked as true if
+   *  - we are not the first page
+   *  - loading has finished
+   */
+  hasNoResultsOnSelectedPage: boolean
+
+  hasNoResultsBecauseOfFilters: boolean
 }
 
-export interface ComprehensivePaginationEngine<Item, QueryResult extends List> {
-  selectedPage: number
-  offsetForQuery: number
-  limitForQuery: number
-  paramsForQuery: { offset: number; limit: number }
-  getResults: (queryResult: QueryResult | undefined, key?: keyof QueryResult) => PaginatedResults<Item>
+/**
+ * A Comprehensive PaginationEngine sits between the server and the consumer of the data and does transformations
+ *
+ * Specifically, the interface for loading the data and the one for the data consumers are decoupled.
+ */
+export interface ComprehensivePaginationEngine<
+  Item,
+  QueryResult extends List,
+  ExtractedData = typeof undefined,
+> {
+  /**
+   * The currently selected page from the data consumer's POV
+   */
+  selectedPageForClient: number
+
+  /**
+   * Parameters for data to be loaded from the server
+   */
+  paramsForServer: { offset: number; limit: number }
+
+  /**
+   * Get the current data/state info for the data consumer component.
+   *
+   * @param isLoading Is the data still being loaded from the server?
+   * @param queryResult the data coming in the server, requested according to this engine's specs, including metadata
+   * @param key The field where the actual records can be found within queryResults
+   */
+  getResults: (
+    isLoading: boolean,
+    isFetched: boolean,
+    queryResult: QueryResult | undefined,
+    key?: keyof QueryResult,
+  ) => ComprehensivePaginatedResults<Item, ExtractedData>
 }

src/app/components/Table/useClientSidePagination.ts

@@ -1,14 +1,58 @@
 import { To, useSearchParams } from 'react-router-dom'
 import { AppErrors } from '../../../types/errors'
-import { ComprehensivePaginationEngine } from './PaginationEngine'
+import { ComprehensivePaginatedResults, ComprehensivePaginationEngine } from './PaginationEngine'
 import { List } from '../../../oasis-nexus/api'
 import { TablePaginationProps } from './TablePagination'
 
-type ClientSizePaginationParams<Item> = {
+type Filter<Item> = (item: Item) => boolean
+
+type ClientSizePaginationParams<Item, QueryResult extends List, ExtractedData = typeof undefined> = {
+  /**
+   * How should we call the query parameter (in the URL)?
+   */
   paramName: string
+
+  /**
+   * The pagination page size from the POV of the data consumer component
+   */
   clientPageSize: number
+
+  /**
+   * The pagination page size used for actually loading the data from the server.
+   *
+   * Please note that currently this engine doesn't handle when the data consumer requires data which is not
+   * part of the initial window on the server side.
+   */
   serverPageSize: number
-  filter?: (item: Item) => boolean
+
+  /**
+   * Filter to be applied to the loaded data.
+   *
+   * This is the order of processing:
+   *  - transform()
+   *  - filter
+   *  - filters
+   *  - order   */
+  filter?: Filter<Item> | undefined
+
+  /**
+   * Filter to be applied to the loaded data.
+   *
+   * This is the order of processing:
+   *  - transform()
+   *  - filter
+   *  - filters
+   *  - order
+   */
+  filters?: (Filter<Item> | undefined)[]
+
+  /**
+   * Transformation to be applied after loading the data from the server, before presenting it to the data consumer component
+   *
+   * Can be used for ordering, aggregation, etc.D
+   * If both transform and filter is set, transform will run first.
+   */
+  transform?: (input: Item[], results: QueryResult) => [Item[], ExtractedData]
 }
 
 const knownListKeys: string[] = ['total_count', 'is_total_count_clipped']
@@ -27,13 +71,21 @@ function findListIn<T extends List, Item>(data: T): Item[] {
   }
 }
 
-export function useClientSidePagination<Item, QueryResult extends List>({
+/**
+ * The ClientSidePagination engine loads the data from the server with a big window in one go, for in-memory pagination
+ */
+export function useClientSidePagination<Item, QueryResult extends List, ExtractedData = typeof undefined>({
   paramName,
   clientPageSize,
   serverPageSize,
   filter,
-}: ClientSizePaginationParams<Item>): ComprehensivePaginationEngine<Item, QueryResult> {
-  const selectedServerPage = 1
+  filters,
+  transform,
+}: ClientSizePaginationParams<Item, QueryResult, ExtractedData>): ComprehensivePaginationEngine<
+  Item,
+  QueryResult,
+  ExtractedData
+> {
   const [searchParams] = useSearchParams()
   const selectedClientPageString = searchParams.get(paramName)
   const selectedClientPage = parseInt(selectedClientPageString ?? '1', 10)
@@ -57,30 +109,51 @@ export function useClientSidePagination<Item, QueryResult extends List>({
     return { search: newSearchParams.toString() }
   }
 
-  const limit = serverPageSize
-  const offset = (selectedServerPage - 1) * clientPageSize
+  // From the server, we always want to load the first batch of data, with the provided (big) window.
+  // In theory, we could move this window as required, but currently this is not implemented.
+  const selectedServerPage = 1
+
+  // The query parameters that should be used for loading the data from the server
   const paramsForQuery = {
-    offset,
-    limit,
+    offset: (selectedServerPage - 1) * serverPageSize,
+    limit: serverPageSize,
   }
 
   return {
-    selectedPage: selectedClientPage,
-    offsetForQuery: offset,
-    limitForQuery: limit,
-    paramsForQuery,
-    getResults: (queryResult, key) => {
-      const data = queryResult
-        ? key
-          ? (queryResult[key] as Item[])
-          : findListIn<QueryResult, Item>(queryResult)
+    selectedPageForClient: selectedClientPage,
+    paramsForServer: paramsForQuery,
+    getResults: (
+      isLoading,
+      isFetched,
+      queryResult,
+      key,
+    ): ComprehensivePaginatedResults<Item, ExtractedData> => {
+      const data = queryResult // we want to get list of items out from the incoming results
+        ? key // do we know where (in which field) to look?
+          ? (queryResult[key] as Item[]) // If yes, just get out the data
+          : findListIn<QueryResult, Item>(queryResult) // If no, we will try to guess
         : undefined
-      const filteredData = !!data && !!filter ? data.filter(filter) : data
 
+      // Apply the specified client-side transformation
+      const [transformedData, extractedData] = !!data && !!transform ? transform(data, queryResult!) : [data]
+
+      // Select the filters to use. (filter field, filters field, drop undefined ones)
+      const filtersToApply = [filter, ...(filters ?? [])].filter(f => !!f) as Filter<Item>[]
+
+      // Apply the specified filtering
+      const filteredData = transformedData
+        ? filtersToApply.reduce<Item[]>(
+            (partiallyFiltered, nextFilter) => partiallyFiltered.filter(nextFilter),
+            transformedData,
+          )
+        : transformedData
+
+      // The data window from the POV of the data consumer component
       const offset = (selectedClientPage - 1) * clientPageSize
       const limit = clientPageSize
       const dataWindow = filteredData ? filteredData.slice(offset, offset + limit) : undefined
 
+      // The control interface for the data consumer component (i.e. Table)
       const tableProps: TablePaginationProps = {
         selectedPage: selectedClientPage,
         linkToPage,
@@ -93,11 +166,25 @@ export function useClientSidePagination<Item, QueryResult extends List>({
         isTotalCountClipped: queryResult?.is_total_count_clipped, // TODO
         rowsPerPage: clientPageSize,
       }
+
+      const isOnFirstPage = tableProps.selectedPage === 1
+      const hasData = !!dataWindow?.length
+      const hasNoResultsOnSelectedPage = !isLoading && !isOnFirstPage && !hasData
+      const hasNoResultsWhatsoever = !isLoading && !queryResult?.total_count
+      const hasNoResultsBecauseOfFilters = !isLoading && !!transformedData?.length && !filteredData?.length
+
       return {
         tablePaginationProps: tableProps,
         data: dataWindow,
+        extractedData,
+        isLoading,
+        isFetched,
+        hasData,
+        isOnFirstPage,
+        hasNoResultsWhatsoever,
+        hasNoResultsOnSelectedPage,
+        hasNoResultsBecauseOfFilters,
       }
     },
-    // tableProps,
   }
 }

src/app/components/Table/useComprehensiveSearchParamsPagination.ts

@@ -1,18 +1,28 @@
 import { To, useSearchParams } from 'react-router-dom'
 import { AppErrors } from '../../../types/errors'
-import { ComprehensivePaginationEngine } from './PaginationEngine'
+import { ComprehensivePaginatedResults, ComprehensivePaginationEngine } from './PaginationEngine'
 import { List } from '../../../oasis-nexus/api'
 import { TablePaginationProps } from './TablePagination'
 
+type Filter<Item> = (item: Item) => boolean
+
 type ComprehensiveSearchParamsPaginationParams<Item> = {
   paramName: string
   pageSize: number
+
   /**
    * @deprecated this will mess up page size.
    *
    * Consider using client-side pagination instead.
    */
-  filter?: (item: Item) => boolean
+  filter?: Filter<Item> | undefined
+
+  /**
+   * @deprecated this will mess up page size.
+   *
+   * Consider using client-side pagination instead.
+   */
+  filters?: (Filter<Item> | undefined)[]
 }
 
 const knownListKeys: string[] = ['total_count', 'is_total_count_clipped']
@@ -35,6 +45,7 @@ export function useComprehensiveSearchParamsPagination<Item, QueryResult extends
   paramName,
   pageSize,
   filter,
+  filters,
 }: ComprehensiveSearchParamsPaginationParams<Item>): ComprehensivePaginationEngine<Item, QueryResult> {
   const [searchParams] = useSearchParams()
   const selectedPageString = searchParams.get(paramName)
@@ -64,27 +75,50 @@ export function useComprehensiveSearchParamsPagination<Item, QueryResult extends
   }
 
   return {
-    selectedPage,
-    offsetForQuery: offset,
-    limitForQuery: limit,
-    paramsForQuery,
-    getResults: (queryResult, key) => {
+    selectedPageForClient: selectedPage,
+    paramsForServer: paramsForQuery,
+    getResults: (isLoading, isFetched, queryResult, key): ComprehensivePaginatedResults<Item> => {
       const data = queryResult
         ? key
           ? (queryResult[key] as Item[])
           : findListIn<QueryResult, Item>(queryResult)
         : undefined
-      const filteredData = !!data && !!filter ? data.filter(filter) : data
+
+      // Select the filters to use. (filter field, filters field, drop undefined ones)
+      const filtersToApply = [filter, ...(filters ?? [])].filter(f => !!f) as Filter<Item>[]
+
+      // Apply the specified filtering
+      const filteredData = data
+        ? filtersToApply.reduce<Item[]>(
+            (partiallyFiltered, nextFilter) => partiallyFiltered.filter(nextFilter),
+            data,
+          )
+        : data
+
       const tableProps: TablePaginationProps = {
         selectedPage,
         linkToPage,
         totalCount: queryResult?.total_count,
         isTotalCountClipped: queryResult?.is_total_count_clipped,
         rowsPerPage: pageSize,
       }
+
+      const isOnFirstPage = tableProps.selectedPage === 1
+      const hasData = !!filteredData?.length
+      const hasNoResultsOnSelectedPage = !isLoading && !isOnFirstPage && !hasData
+      const hasNoResultsWhatsoever = !isLoading && !queryResult?.total_count
+      const hasNoResultsBecauseOfFilters = !isLoading && !!data?.length && !filteredData?.length
+
       return {
         tablePaginationProps: tableProps,
         data: filteredData,
+        isLoading,
+        isFetched,
+        hasData,
+        isOnFirstPage,
+        hasNoResultsOnSelectedPage,
+        hasNoResultsWhatsoever,
+        hasNoResultsBecauseOfFilters,
       }
     },
     // tableProps,