Fix row rendering in infinite scroll

changelogs/fragments/8060.yml

@@ -0,0 +1,2 @@
+fix:
+- Fix row rendering in Discover infinite scroll ([#8060](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8060))

src/plugins/discover/public/application/components/default_discover_table/default_discover_table.test.tsx

@@ -0,0 +1,158 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import React from 'react';
+import { IntlProvider } from 'react-intl';
+import { act, render, waitFor, screen } from '@testing-library/react';
+import { DefaultDiscoverTable } from './default_discover_table';
+import { OpenSearchSearchHit } from '../../doc_views/doc_views_types';
+import { coreMock } from '../../../../../../core/public/mocks';
+import { getStubIndexPattern } from '../../../../../data/public/test_utils';
+
+jest.mock('../../../opensearch_dashboards_services', () => ({
+  getServices: jest.fn().mockReturnValue({
+    uiSettings: {
+      get: jest.fn().mockImplementation((key) => {
+        switch (key) {
+          case 'discover:sampleSize':
+            return 100;
+          case 'shortDots:enable':
+            return true;
+          case 'doc_table:hideTimeColumn':
+            return false;
+          case 'discover:sort:defaultOrder':
+            return 'desc';
+          default:
+            return null;
+        }
+      }),
+    },
+  }),
+}));
+
+describe('DefaultDiscoverTable', () => {
+  const indexPattern = getStubIndexPattern(
+    'test-index-pattern',
+    (cfg) => cfg,
+    '@timestamp',
+    [
+      { name: 'textField', type: 'text' },
+      { name: 'longField', type: 'long' },
+      { name: '@timestamp', type: 'date' },
+    ],
+    coreMock.createSetup()
+  );
+
+  // Generate 50 hits with sample fields
+  const hits = [...Array(100).keys()].map((key) => {
+    return {
+      _id: key.toString(),
+      fields: {
+        textField: `value${key}`,
+        longField: key,
+        '@timestamp': new Date((1720000000 + key) * 1000),
+      },
+    };
+  });
+
+  const getDefaultDiscoverTable = (hitsOverride?: OpenSearchSearchHit[]) => (
+    <IntlProvider locale="en">
+      <DefaultDiscoverTable
+        columns={['textField', 'longField', '@timestamp']}
+        rows={(hitsOverride ?? hits) as OpenSearchSearchHit[]}
+        indexPattern={indexPattern}
+        sort={[]}
+        onSort={jest.fn()}
+        onRemoveColumn={jest.fn()}
+        onMoveColumn={jest.fn()}
+        onAddColumn={jest.fn()}
+        onFilter={jest.fn()}
+      />
+    </IntlProvider>
+  );
+
+  let intersectionObserverCallback: (entries: IntersectionObserverEntry[]) => void = (_) => {};
+  const mockIntersectionObserver = jest.fn();
+
+  beforeEach(() => {
+    mockIntersectionObserver.mockImplementation((...args) => {
+      intersectionObserverCallback = args[0];
+      return {
+        observe: () => null,
+        unobserve: () => null,
+        disconnect: () => null,
+      };
+    });
+    window.IntersectionObserver = mockIntersectionObserver;
+  });
+
+  it('should render the correct number of rows initially', () => {
+    const { container } = render(getDefaultDiscoverTable());
+
+    const tableRows = container.querySelectorAll('tbody tr');
+    expect(tableRows.length).toBe(10);
+  });
+
+  it('should load more rows when scrolling to the bottom', async () => {
+    const { container } = render(getDefaultDiscoverTable());
+
+    const sentinel = container.querySelector('div[data-test-subj="discoverRenderedRowsProgress"]');
+    const mockScrollEntry = { isIntersecting: true, target: sentinel };
+    act(() => {
+      intersectionObserverCallback([mockScrollEntry] as IntersectionObserverEntry[]);
+    });
+
+    await waitFor(() => {
+      const tableRows = container.querySelectorAll('tbody tr');
+      expect(tableRows.length).toBe(20);
+    });
+  });
+
+  it('should display the sample size callout when all rows are rendered', async () => {
+    const { container } = render(getDefaultDiscoverTable());
+
+    let sentinel = container.querySelector('div[data-test-subj="discoverRenderedRowsProgress"]');
+
+    // Simulate scrolling to the bottom until all rows are rendered
+    while (sentinel) {
+      const mockScrollEntry = { isIntersecting: true, target: sentinel };
+      act(() => {
+        intersectionObserverCallback([mockScrollEntry] as IntersectionObserverEntry[]);
+      });
+      sentinel = container.querySelector('div[data-test-subj="discoverRenderedRowsProgress"]');
+    }
+
+    await waitFor(() => {
+      const callout = screen.getByTestId('discoverDocTableFooter');
+      expect(callout).toBeInTheDocument();
+    });
+  });
+
+  it('Should restart rendering when new data is available', async () => {
+    const truncHits = hits.slice(0, 35) as OpenSearchSearchHit[];
+    const { container, rerender } = render(getDefaultDiscoverTable(truncHits));
+
+    let sentinel = container.querySelector('div[data-test-subj="discoverRenderedRowsProgress"]');
+
+    // Keep scrolling until all the current rows are exhausted
+    while (sentinel) {
+      const mockScrollEntry = { isIntersecting: true, target: sentinel };
+      act(() => {
+        intersectionObserverCallback([mockScrollEntry] as IntersectionObserverEntry[]);
+      });
+      sentinel = container.querySelector('div[data-test-subj="discoverRenderedRowsProgress"]');
+    }
+
+    // Make the other rows available
+    rerender(getDefaultDiscoverTable(hits as OpenSearchSearchHit[]));
+
+    await waitFor(() => {
+      const progressSentinel = container.querySelector(
+        'div[data-test-subj="discoverRenderedRowsProgress"]'
+      );
+      expect(progressSentinel).toBeInTheDocument();
+    });
+  });
+});

src/plugins/discover/public/application/components/default_discover_table/default_discover_table.tsx

@@ -41,6 +41,8 @@
 // ToDo: These would need to be read from an upcoming config panel
 const PAGINATED_PAGE_SIZE = 50;
 const INFINITE_SCROLLED_PAGE_SIZE = 10;
+// How far to queue unrendered rows ahead of time during infinite scrolling
+const DESIRED_ROWS_LOOKAHEAD = 5 * INFINITE_SCROLLED_PAGE_SIZE;
 
 const DefaultDiscoverTableUI = ({
   columns,
@@ -86,7 +88,7 @@
    */
   const [renderedRowCount, setRenderedRowCount] = useState(INFINITE_SCROLLED_PAGE_SIZE);
   const [desiredRowCount, setDesiredRowCount] = useState(
-    Math.min(rows.length, 5 * INFINITE_SCROLLED_PAGE_SIZE)
+    Math.min(rows.length, DESIRED_ROWS_LOOKAHEAD)
   );
   const [displayedRows, setDisplayedRows] = useState(rows.slice(0, PAGINATED_PAGE_SIZE));
   const [currentRowCounts, setCurrentRowCounts] = useState({
@@ -118,10 +120,14 @@
           if (entries[0].isIntersecting) {
             // Load another batch of rows, some immediately and some lazily
             setRenderedRowCount((prevRowCount) => prevRowCount + INFINITE_SCROLLED_PAGE_SIZE);
-            setDesiredRowCount((prevRowCount) => prevRowCount + 5 * INFINITE_SCROLLED_PAGE_SIZE);
+            setDesiredRowCount((prevRowCount) => prevRowCount + DESIRED_ROWS_LOOKAHEAD);
           }
         },
-        { threshold: 1.0 }
+        {
+          // Important that 0 < threshold < 1, since there OSD application div has a transparent
+          // fade at the bottom which causes the sentinel element to sometimes not be 100% visible
+          threshold: 0.1,
+        }
       );
 
       observerRef.current.observe(sentinelElement);
@@ -155,6 +161,10 @@
   const lazyLoadRequestFrameRef = useRef<number>(0);
   const lazyLoadLastTimeRef = useRef<number>(0);
 
+  // When doing infinite scrolling, the `rows` prop gets regularly updated from the outside: we only
+  // render the additional rows when we know the load isn't too high. To prevent overloading the
+  // renderer, we throttle by current framerate and only render if the frames are fast enough, then
+  // we increase the rendered row count and trigger a re-render.
   React.useEffect(() => {
     if (!showPagination) {
       const loadMoreRows = (time: number) => {
@@ -254,7 +264,16 @@
         </table>
         {!showPagination && renderedRowCount < rows.length && (
           <div ref={sentinelRef}>
-            <EuiProgress size="xs" color="accent" />
+            <EuiProgress
+              size="xs"
+              color="accent"
+              data-test-subj="discoverRenderedRowsProgress"
+              style={{
+                // Add a little margin if we aren't rendering the truncation callout below, to make
+                // the progress bar render better when it's not present
+                marginBottom: rows.length !== sampleSize ? '5px' : '0',
+              }}
+            />
           </div>
         )}
         {!showPagination && rows.length === sampleSize && (