feat(tree): improve Tree Data speed considerably

examples/webpack-demo-vanilla-bundle/src/examples/example05.html

@@ -15,28 +15,39 @@ <h6 class="title is-6 italic">
 </h6>
 <div class="columns">
   <div class="column is-narrow">
-    <button onclick.delegate="addNewRow()" class="button is-small is-info">
-      <span class="icon mdi mdi-plus"></span>
-      <span>Add New Item (in 1st group)</span>
-    </button>
-    <button onclick.delegate="collapseAll()" data-test="collapse-all" class="button is-small">
-      <span class="icon mdi mdi-arrow-collapse"></span>
-      <span>Collapse All</span>
-    </button>
-    <button onclick.delegate="expandAll()" data-test="expand-all" class="button is-small">
-      <span class="icon mdi mdi-arrow-expand"></span>
-      <span>Expand All</span>
-    </button>
-    <button onclick.delegate="logFlatStructure()" class="button is-small">
-      <span>Log Flat Structure</span>
-    </button>
-    <button onclick.delegate="logHierarchicalStructure()" class="button is-small">
-      <span>Log Hierarchical Structure</span>
-    </button>
-    <button onclick.delegate="dynamicallyChangeFilter()" class="button is-small">
-      <span class="icon mdi mdi-filter-outline"></span>
-      <span>Dynamically Change Filter (% complete &lt; 40)</span>
-    </button>
+    <div class="row" style="margin-bottom: 4px;">
+      <button class="button is-small" data-test="add-500-rows-btn" onclick.delegate="loadData(500)">
+        500 rows
+      </button>
+      <button class="button is-small" data-test="add-50k-rows-btn" onclick.delegate="loadData(50000)">
+        50k rows
+      </button>
+      <button onclick.delegate="dynamicallyChangeFilter()" class="button is-small">
+        <span class="icon mdi mdi-filter-outline"></span>
+        <span>Dynamically Change Filter (% complete &lt; 40)</span>
+      </button>
+    </div>
+
+    <div class="row" style="margin-bottom: 4px;">
+      <button onclick.delegate="addNewRow()" class="button is-small is-info">
+        <span class="icon mdi mdi-plus"></span>
+        <span>Add New Item (in 1st group)</span>
+      </button>
+      <button onclick.delegate="collapseAll()" data-test="collapse-all" class="button is-small">
+        <span class="icon mdi mdi-arrow-collapse"></span>
+        <span>Collapse All</span>
+      </button>
+      <button onclick.delegate="expandAll()" data-test="expand-all" class="button is-small">
+        <span class="icon mdi mdi-arrow-expand"></span>
+        <span>Expand All</span>
+      </button>
+      <button onclick.delegate="logFlatStructure()" class="button is-small">
+        <span>Log Flat Structure</span>
+      </button>
+      <button onclick.delegate="logHierarchicalStructure()" class="button is-small">
+        <span>Log Hierarchical Structure</span>
+      </button>
+    </div>
   </div>
 </div>
 

examples/webpack-demo-vanilla-bundle/src/examples/example05.ts

@@ -11,7 +11,7 @@ import { Slicker, SlickVanillaGridBundle } from '@slickgrid-universal/vanilla-bu
 import { ExampleGridOptions } from './example-grid-options';
 import './example05.scss';
 
-const NB_ITEMS = 200;
+const NB_ITEMS = 500;
 
 export class Example5 {
   columnDefinitions: Column[];
@@ -26,8 +26,8 @@ export class Example5 {
     const gridContainerElm = document.querySelector<HTMLDivElement>('.grid5');
 
     this.sgb = new Slicker.GridBundle(gridContainerElm, this.columnDefinitions, { ...ExampleGridOptions, ...this.gridOptions });
-    this.dataset = this.mockDataset();
-    this.sgb.dataset = this.dataset;
+    this.dataset = this.loadData(NB_ITEMS);
+    // this.sgb.dataset = this.dataset;
   }
 
   dispose() {
@@ -93,10 +93,12 @@ export class Example5 {
       enableTreeData: true, // you must enable this flag for the filtering & sorting to work as expected
       treeDataOptions: {
         columnId: 'title',
-        // levelPropName: 'indent', // this is optional, you can define the tree level property name that will be used for the sorting/indentation, internally it will use "__treeLevel"
         parentPropName: 'parentId',
+        // this is optional, you can define the tree level property name that will be used for the sorting/indentation, internally it will use "__treeLevel"
+        // levelPropName: 'indent',
 
         // you can optionally sort by a different column and/or sort direction
+        // this is the recommend approach, unless you are 100% that your original array is already sorted (in most cases it's not)
         initialSort: {
           columnId: 'title',
           direction: 'ASC'
@@ -160,17 +162,17 @@ export class Example5 {
     console.log('flat array', this.sgb.treeDataService.dataset);
   }
 
-  mockDataset() {
+  loadData(rowCount: number) {
     let indent = 0;
     const parents = [];
     const data = [];
 
     // prepare the data
-    for (let i = 0; i < NB_ITEMS; i++) {
+    for (let i = 0; i < rowCount; i++) {
       const randomYear = 2000 + Math.floor(Math.random() * 10);
       const randomMonth = Math.floor(Math.random() * 11);
       const randomDay = Math.floor((Math.random() * 29));
-      const d = (data[i] = {});
+      const item = (data[i] = {});
       let parentId;
 
       // for implementing filtering/sorting, don't go over indent of 2
@@ -188,15 +190,19 @@ export class Example5 {
         parentId = null;
       }
 
-      d['id'] = i;
-      d['parentId'] = parentId;
-      // d['title'] = `Task ${i}    -   [P]: ${parentId}`;
-      d['title'] = `Task ${i}`;
-      d['duration'] = '5 days';
-      d['percentComplete'] = Math.round(Math.random() * 100);
-      d['start'] = new Date(randomYear, randomMonth, randomDay);
-      d['finish'] = new Date(randomYear, (randomMonth + 1), randomDay);
-      d['effortDriven'] = (i % 5 === 0);
+      item['id'] = i;
+      // item['__treeLevel'] = indent;
+      item['parentId'] = parentId;
+      // item['title'] = `Task ${i}`;
+      item['title'] = `<span style="font-weight:500">Task ${i}</span>  <span style="font-size:11px; margin-left: 15px;">(parentId: ${parentId})</span>`;
+      item['duration'] = '5 days';
+      item['percentComplete'] = Math.round(Math.random() * 100);
+      item['start'] = new Date(randomYear, randomMonth, randomDay);
+      item['finish'] = new Date(randomYear, (randomMonth + 1), randomDay);
+      item['effortDriven'] = (i % 5 === 0);
+    }
+    if (this.sgb) {
+      this.sgb.dataset = data;
     }
     return data;
   }

packages/common/package.json

@@ -70,7 +70,8 @@
     "jquery-ui-dist": "^1.12.1",
     "moment-mini": "^2.24.0",
     "multiple-select-modified": "^1.3.11",
-    "slickgrid": "^2.4.34"
+    "slickgrid": "^2.4.34",
+    "un-flatten-tree": "^2.0.12"
   },
   "devDependencies": {
     "@types/dompurify": "^2.2.2",

packages/common/src/formatters/__tests__/treeExportFormatter.spec.ts

@@ -32,7 +32,7 @@ describe('Tree Export Formatter', () => {
 
   it('should throw an error when oarams are mmissing', () => {
     expect(() => treeExportFormatter(1, 1, 'blah', {} as Column, {}, gridStub))
-      .toThrowError('You must provide valid "treeDataOptions" in your Grid Options and it seems that there are no tree level found in this row');
+      .toThrowError('[Slickgrid-Universal] You must provide valid "treeDataOptions" in your Grid Options, however it seems that we could not find any tree level info on the current item datacontext row.');
   });
 
   it('should return empty string when DataView is not correctly formed', () => {

packages/common/src/formatters/__tests__/treeFormatter.spec.ts

@@ -32,7 +32,7 @@ describe('Tree Formatter', () => {
 
   it('should throw an error when oarams are mmissing', () => {
     expect(() => treeFormatter(1, 1, 'blah', {} as Column, {}, gridStub))
-      .toThrowError('You must provide valid "treeDataOptions" in your Grid Options and it seems that there are no tree level found in this row');
+      .toThrowError('[Slickgrid-Universal] You must provide valid "treeDataOptions" in your Grid Options, however it seems that we could not find any tree level info on the current item datacontext row.');
   });
 
   it('should return empty string when DataView is not correctly formed', () => {

packages/common/src/formatters/treeExportFormatter.ts

@@ -25,7 +25,7 @@ export const treeExportFormatter: Formatter = (_row, _cell, value, columnDef, da
   }
 
   if (!dataContext.hasOwnProperty(treeLevelPropName)) {
-    throw new Error('You must provide valid "treeDataOptions" in your Grid Options and it seems that there are no tree level found in this row');
+    throw new Error('[Slickgrid-Universal] You must provide valid "treeDataOptions" in your Grid Options, however it seems that we could not find any tree level info on the current item datacontext row.');
   }
 
   if (dataView?.getItemByIdx) {

packages/common/src/formatters/treeFormatter.ts

@@ -1,10 +1,10 @@
 import { SlickDataView, Formatter } from './../interfaces/index';
-import { getDescendantProperty, htmlEncode } from '../services/utilities';
+import { getDescendantProperty, sanitizeTextByAvailableSanitizer } from '../services/utilities';
 
 /** Formatter that must be use with a Tree Data column */
 export const treeFormatter: Formatter = (_row, _cell, value, columnDef, dataContext, grid) => {
-  const dataView = grid?.getData<SlickDataView>();
-  const gridOptions = grid?.getOptions();
+  const dataView = grid.getData<SlickDataView>();
+  const gridOptions = grid.getOptions();
   const treeDataOptions = gridOptions?.treeDataOptions;
   const treeLevelPropName = treeDataOptions?.levelPropName ?? '__treeLevel';
   const indentMarginLeft = treeDataOptions?.indentMarginLeft ?? 15;
@@ -23,27 +23,25 @@ export const treeFormatter: Formatter = (_row, _cell, value, columnDef, dataCont
   }
 
   if (!dataContext.hasOwnProperty(treeLevelPropName)) {
-    throw new Error('You must provide valid "treeDataOptions" in your Grid Options and it seems that there are no tree level found in this row');
+    throw new Error('[Slickgrid-Universal] You must provide valid "treeDataOptions" in your Grid Options, however it seems that we could not find any tree level info on the current item datacontext row.');
   }
 
   if (dataView?.getItemByIdx) {
-    if (typeof outputValue === 'string') {
-      outputValue = htmlEncode(outputValue);
-    }
-    const identifierPropName = dataView.getIdPropertyName() || 'id';
+    const sanitizedOutputValue = sanitizeTextByAvailableSanitizer(gridOptions, outputValue);
+    const identifierPropName = dataView.getIdPropertyName() ?? 'id';
     const treeLevel = dataContext[treeLevelPropName] || 0;
     const spacer = `<span style="display:inline-block; width:${indentMarginLeft * treeLevel}px;"></span>`;
     const idx = dataView.getIdxById(dataContext[identifierPropName]);
     const nextItemRow = dataView.getItemByIdx((idx || 0) + 1);
 
     if (nextItemRow?.[treeLevelPropName] > treeLevel) {
       if (dataContext.__collapsed) {
-        return `${spacer}<span class="slick-group-toggle collapsed"></span>&nbsp;${outputValue}`;
+        return `${spacer}<span class="slick-group-toggle collapsed"></span>&nbsp;${sanitizedOutputValue}`;
       } else {
-        return `${spacer}<span class="slick-group-toggle expanded"></span>&nbsp;${outputValue}`;
+        return `${spacer}<span class="slick-group-toggle expanded"></span>&nbsp;${sanitizedOutputValue}`;
       }
     }
-    return `${spacer}<span class="slick-group-toggle"></span>&nbsp;${outputValue}`;
+    return `${spacer}<span class="slick-group-toggle"></span>&nbsp;${sanitizedOutputValue}`;
   }
   return '';
 };

packages/common/src/interfaces/treeDataOption.interface.ts

@@ -32,9 +32,6 @@ export interface TreeDataOption {
   /** Defaults to "__treeLevel", object property name used to designate the Tree Level depth number */
   levelPropName?: string;
 
-  /** Defaults to "__hasChildren", object property name used to designate if the item has Children flag */
-  hasChildrenFlagPropName?: string;
-
   /**
    * Defaults to 15px, margin to add from the left (calculated by the tree level multiplied by this number).
    * For example if tree depth level is 2, the calculation will be (2 * 15 = 30), so the column will be displayed 30px from the left

packages/common/src/services/__tests__/grid.service.spec.ts

@@ -86,7 +86,7 @@ const paginationServiceStub = {
 } as unknown as PaginationService;
 
 const treeDataServiceStub = {
-  convertFlatDatasetConvertToHierarhicalView: jest.fn(),
+  convertFlatToHierarchicalDataset: jest.fn(),
   init: jest.fn(),
   convertToHierarchicalDatasetAndSort: jest.fn(),
   dispose: jest.fn(),
@@ -811,7 +811,7 @@ describe('Grid Service', () => {
 
       jest.spyOn(dataviewStub, 'getItems').mockReturnValue(mockFlatDataset);
       jest.spyOn(dataviewStub, 'getRowById').mockReturnValue(0);
-      jest.spyOn(treeDataServiceStub, 'convertToHierarchicalDatasetAndSort').mockReturnValue({ flat: mockFlatDataset, hierarchical: mockHierarchical });
+      jest.spyOn(treeDataServiceStub, 'convertToHierarchicalDatasetAndSort').mockReturnValue({ flat: mockFlatDataset as any[], hierarchical: mockHierarchical as any[] });
       jest.spyOn(gridStub, 'getOptions').mockReturnValue({ enableAutoResize: true, enableRowSelection: true, enableTreeData: true } as GridOption);
       jest.spyOn(SharedService.prototype, 'allColumns', 'get').mockReturnValue(mockColumns);
       const setItemSpy = jest.spyOn(dataviewStub, 'setItems');
@@ -836,7 +836,7 @@ describe('Grid Service', () => {
 
       jest.spyOn(dataviewStub, 'getItems').mockReturnValue(mockFlatDataset);
       jest.spyOn(dataviewStub, 'getRowById').mockReturnValue(0);
-      jest.spyOn(treeDataServiceStub, 'convertToHierarchicalDatasetAndSort').mockReturnValue({ flat: mockFlatDataset, hierarchical: mockHierarchical });
+      jest.spyOn(treeDataServiceStub, 'convertToHierarchicalDatasetAndSort').mockReturnValue({ flat: mockFlatDataset as any[], hierarchical: mockHierarchical as any[] });
       jest.spyOn(gridStub, 'getOptions').mockReturnValue({ enableAutoResize: true, enableRowSelection: true, enableTreeData: true } as GridOption);
       jest.spyOn(SharedService.prototype, 'allColumns', 'get').mockReturnValue(mockColumns);
       const setItemSpy = jest.spyOn(dataviewStub, 'setItems');

packages/common/src/services/__tests__/treeData.service.spec.ts

@@ -87,6 +87,16 @@ describe('TreeData Service', () => {
     }
   });
 
+  it('should throw an error when used without filter grid option', (done) => {
+    try {
+      gridOptionsMock.multiColumnSort = true;
+      service.init(gridStub);
+    } catch (e) {
+      expect(e.toString()).toContain('[Slickgrid-Universal] It looks like you are trying to use Tree Data without using the filtering option');
+      done();
+    }
+  });
+
   it('should throw an error when enableTreeData is enabled with Pagination since that is not supported', (done) => {
     try {
       gridOptionsMock.enablePagination = true;
@@ -293,7 +303,7 @@ describe('TreeData Service', () => {
         }];
         const setSortSpy = jest.spyOn(gridStub, 'setSortColumns');
         jest.spyOn(gridStub, 'getColumnIndex').mockReturnValue(0);
-        jest.spyOn(sortServiceStub, 'sortHierarchicalDataset').mockReturnValue({ flat: mockFlatDataset, hierarchical: mockHierarchical });
+        jest.spyOn(sortServiceStub, 'sortHierarchicalDataset').mockReturnValue({ flat: mockFlatDataset as any[], hierarchical: mockHierarchical as any[] });
 
         service.init(gridStub);
         const result = service.convertToHierarchicalDatasetAndSort(mockFlatDataset, mockColumns, gridOptionsMock);
@@ -303,7 +313,7 @@ describe('TreeData Service', () => {
           sortAsc: true,
           sortCol: mockColumns[0]
         }]);
-        expect(result).toEqual({ flat: mockFlatDataset, hierarchical: mockHierarchical });
+        expect(result).toEqual({ flat: mockFlatDataset as any[], hierarchical: mockHierarchical as any[] });
       });
 
       it('should sort by the Tree column by the "initialSort" provided', () => {
@@ -318,7 +328,7 @@ describe('TreeData Service', () => {
         }];
         const setSortSpy = jest.spyOn(gridStub, 'setSortColumns');
         jest.spyOn(gridStub, 'getColumnIndex').mockReturnValue(0);
-        jest.spyOn(sortServiceStub, 'sortHierarchicalDataset').mockReturnValue({ flat: mockFlatDataset, hierarchical: mockHierarchical });
+        jest.spyOn(sortServiceStub, 'sortHierarchicalDataset').mockReturnValue({ flat: mockFlatDataset as any[], hierarchical: mockHierarchical as any[] });
 
         service.init(gridStub);
         const result = service.convertToHierarchicalDatasetAndSort(mockFlatDataset, mockColumns, gridOptionsMock);
@@ -328,7 +338,7 @@ describe('TreeData Service', () => {
           sortAsc: false,
           sortCol: mockColumns[1]
         }]);
-        expect(result).toEqual({ flat: mockFlatDataset, hierarchical: mockHierarchical });
+        expect(result).toEqual({ flat: mockFlatDataset as any[], hierarchical: mockHierarchical as any[] });
       });
     });
 

packages/common/src/services/__tests__/utilities.spec.ts

@@ -2,7 +2,7 @@ import 'jest-extended';
 import { of } from 'rxjs';
 
 import { FieldType, OperatorType } from '../../enums/index';
-import { GridOption, Subscription } from '../../interfaces/index';
+import { EventSubscription, GridOption } from '../../interfaces/index';
 import { RxJsResourceStub } from '../../../../../test/rxjsResourceStub';
 import {
   addToArrayWhenNotExists,
@@ -86,6 +86,10 @@ describe('Service/Utilies', () => {
     it('should return the a simple string with x spaces only where x is the number of spaces provided as argument', () => {
       expect(addWhiteSpaces(5)).toBe('     ');
     });
+
+    it('should return the a simple html string with x   separator where x is the number of spaces provided as argument', () => {
+      expect(addWhiteSpaces(2)).toBe('  ');
+    });
   });
 
   describe('arrayRemoveItemByIndex method', () => {
@@ -1488,7 +1492,7 @@ describe('Service/Utilies', () => {
     });
 
     it('should be able to unsubscribe all Observables', () => {
-      const subscriptions: Subscription[] = [];
+      const subscriptions: EventSubscription[] = [];
       const observable1 = of([1, 2]);
       const observable2 = of([1, 2]);
       subscriptions.push(observable1.subscribe(), observable2.subscribe());

packages/common/src/services/filter.service.ts

@@ -691,11 +691,12 @@ export class FilterService {
   /**
    * when we have a Filter Presets on a Tree Data View grid, we need to call the pre-filtering of tree data
    * we need to do this because Tree Data is the only type of grid that requires a pre-filter (preFilterTreeData) to be executed before the final filtering
-   * @param filters
+   * @param {Array<Object>} [items] - optional flat array of parent/child items to use while redoing the full sort & refresh
    */
-  refreshTreeDataFilters() {
+  refreshTreeDataFilters(items?: any[]) {
+    const inputItems = items ?? this._dataView.getItems();
     if (this._dataView && this._gridOptions?.enableTreeData) {
-      this._tmpPreFilteredData = this.preFilterTreeData(this._dataView.getItems(), this._columnFilters);
+      this._tmpPreFilteredData = this.preFilterTreeData(inputItems, this._columnFilters);
       this._dataView.refresh(); // and finally this refresh() is what triggers a DataView filtering check
     }
   }

packages/common/src/services/grid.service.ts

@@ -870,16 +870,18 @@ export class GridService {
   }
 
   /**
-   * When dealing with hierarchical dataset, we can invalidate all the rows and force a resort & re-render of the hierarchical dataset.
+   * When dealing with hierarchical (tree) dataset, we can invalidate all the rows and force a full resort & re-render of the hierarchical tree dataset.
    * This method will automatically be called anytime user called `addItem()` or `addItems()`.
    * However please note that it won't be called when `updateItem`, if the data that gets updated does change the tree data column then you should call this method.
+   * @param {Array<Object>} [items] - optional flat array of parent/child items to use while redoing the full sort & refresh
    */
-  invalidateHierarchicalDataset() {
+  invalidateHierarchicalDataset(items?: any[]) {
     // if we add/remove item(s) from the dataset, we need to also refresh our tree data filters
     if (this._gridOptions?.enableTreeData && this.treeDataService) {
-      const sortedDatasetResult = this.treeDataService.convertToHierarchicalDatasetAndSort(this._dataView.getItems(), this.sharedService.allColumns, this._gridOptions);
+      const inputItems = items ?? this._dataView.getItems();
+      const sortedDatasetResult = this.treeDataService.convertToHierarchicalDatasetAndSort(inputItems, this.sharedService.allColumns, this._gridOptions);
       this.sharedService.hierarchicalDataset = sortedDatasetResult.hierarchical;
-      this.filterService.refreshTreeDataFilters();
+      this.filterService.refreshTreeDataFilters(items);
       this._dataView.setItems(sortedDatasetResult.flat);
     }
   }