refetch tileset metadata once dataset is added to the state; refactor…

…ing;

Signed-off-by: Ihor Dykhta <[email protected]>

src/components/src/hooks/use-fetch-vector-tile-metadata.ts

@@ -3,10 +3,11 @@
 
 import {useCallback, useEffect, useState} from 'react';
 
-import {/*MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
+import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
 import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
 
-import {getMVTMetadata, VectorTileMetadata, VectorTileType} from '@kepler.gl/layers';
+import {VectorTileType} from '@kepler.gl/constants';
+import {getMVTMetadata, VectorTileMetadata} from '@kepler.gl/table';
 
 type FetchVectorTileMetadataProps = {
   url: string | null;

src/components/src/modals/export-data-modal.tsx

@@ -2,8 +2,12 @@
 // Copyright contributors to the kepler.gl project
 
 import React, {Component} from 'react';
+import {injectIntl, IntlShape} from 'react-intl';
+
+import {DatasetType, EXPORT_DATA_TYPE_OPTIONS} from '@kepler.gl/constants';
+import {FormattedMessage} from '@kepler.gl/localization';
+import {Datasets} from '@kepler.gl/table';
 
-import {EXPORT_DATA_TYPE_OPTIONS} from '@kepler.gl/constants';
 import {FileType} from '../common/icons';
 import {
   StyledExportSection,
@@ -13,10 +17,6 @@ import {
   CheckMark
 } from '../common/styled-components';
 import {StyledWarning} from './export-map-modal/components';
-import {injectIntl, IntlShape} from 'react-intl';
-import {DatasetType} from '@kepler.gl/layers';
-import {FormattedMessage} from '@kepler.gl/localization';
-import {Datasets} from '@kepler.gl/table';
 
 const getDataRowCount = (
   datasets: Datasets,

src/components/src/modals/tilesets-modals/common.tsx

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-import {VectorTileMetadata} from '@kepler.gl/layers';
+import {VectorTileMetadata} from '@kepler.gl/table';
 
 export type DatasetCreationAttributes = {
   name: string;

src/components/src/modals/tilesets-modals/tileset-vector-form.tsx

@@ -4,13 +4,8 @@
 import React, {useCallback, useEffect, useState, useMemo} from 'react';
 import styled from 'styled-components';
 
-import {
-  getMetaUrl,
-  parseVectorMetadata,
-  VectorTileMetadata,
-  DatasetType,
-  VectorTileType
-} from '@kepler.gl/layers';
+import {DatasetType, VectorTileType} from '@kepler.gl/constants';
+import {getMetaUrl, parseVectorMetadata, VectorTileMetadata} from '@kepler.gl/table';
 import {TileJSON} from '@loaders.gl/mvt';
 import {PMTilesMetadata} from '@loaders.gl/pmtiles';
 

src/components/src/side-panel/common/dataset-info.tsx

@@ -5,7 +5,7 @@ import React from 'react';
 import styled from 'styled-components';
 import {format} from 'd3-format';
 
-import {DatasetType} from '@kepler.gl/layers';
+import {DatasetType} from '@kepler.gl/constants';
 import {FormattedMessage} from '@kepler.gl/localization';
 import {DataContainerInterface} from '@kepler.gl/utils';
 

src/components/src/side-panel/layer-panel/layer-configurator.tsx

@@ -27,20 +27,13 @@ import VisConfigSliderFactory from './vis-config-slider';
 import VisConfigSwitchFactory from './vis-config-switch';
 import VectorTileLayerConfiguratorFactory from './vector-tile-layer-configurator';
 
-import {capitalizeFirstLetter} from '@kepler.gl/utils';
-
+import {ActionHandler, toggleModal} from '@kepler.gl/actions';
 import {AGGREGATION_TYPE_OPTIONS, LAYER_TYPES} from '@kepler.gl/constants';
-import {
-  AggregationLayer,
-  Layer,
-  LayerBaseConfig,
-  VisualChannel,
-  matchDatasetType
-} from '@kepler.gl/layers';
+import {AggregationLayer, Layer, LayerBaseConfig, VisualChannel} from '@kepler.gl/layers';
 
-import {ActionHandler, toggleModal} from '@kepler.gl/actions';
-import {Datasets} from '@kepler.gl/table';
+import {matchDatasetType, Datasets} from '@kepler.gl/table';
 import {ColorUI, LayerVisConfig, NestedPartial} from '@kepler.gl/types';
+import {capitalizeFirstLetter} from '@kepler.gl/utils';
 
 type LayerConfiguratorProps = {
   layer: Layer;

src/constants/src/dataset.ts

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+export enum DatasetType {
+  LOCAL = 'local',
+  VECTOR_TILE = 'vectorTile'
+}
+
+export enum TileType {
+  VECTOR_TILE = 'vectorTile'
+}
+
+export enum VectorTileType {
+  MVT = 'mvt',
+  PMTILES = 'pmtiles'
+}
+
+export type VectorTileDatasetMetadata = {
+  type: VectorTileType;
+  tilesetDataUrl: string;
+  tilesetMetadataUrl?: string;
+};

src/constants/src/index.ts

@@ -7,6 +7,7 @@ export * from './colors-by-theme';
 export * from './color-palettes';
 export * from './default-settings';
 export * from './layers';
+export * from './dataset';
 export {default as KeyEvent} from './keyevent';
 export * from './tooltip';
 export * from './user-feedbacks';

src/layers/src/index.ts

@@ -42,21 +42,8 @@ export {getS2Center} from './s2-geometry-layer/s2-utils';
 export {default as AggregationLayer} from './aggregation-layer';
 import {default as VectorTileLayer} from './vector-tile/vector-tile-layer';
 
-export {
-  getMetaUrl,
-  parseVectorMetadata,
-  matchDatasetType,
-  DatasetType,
-  TileType,
-  VectorTileType
-} from './vector-tile/utils/vector-tile-utils';
-export type {
-  VectorTileDatasetMetadata,
-  VectorTileMetadata
-} from './vector-tile/utils/vector-tile-utils';
 export {default as VectorTileIcon} from './vector-tile/vector-tile-icon';
 export {default as VectorTileLayer} from './vector-tile/vector-tile-layer';
-export {getMVTMetadata} from './vector-tile/mvt-source-utils';
 
 import {LAYER_TYPES} from '@kepler.gl/constants';
 export {parseGeoJsonRawFeature} from './geojson-layer/geojson-utils';

src/layers/src/vector-tile/abstract-tile-layer.ts

@@ -6,6 +6,7 @@ import throttle from 'lodash.throttle';
 import {notNullorUndefined} from '@kepler.gl/common-utils';
 import {SCALE_TYPES, ALL_FIELD_TYPES, LAYER_VIS_CONFIGS} from '@kepler.gl/constants';
 import {
+  isTileDataset,
   KeplerTable as KeplerDataset,
   Datasets as KeplerDatasets,
   GpuFilter
@@ -37,7 +38,6 @@ import {
   VisualChannelDomain,
   VisualChannelField
 } from '../base-layer';
-import {isTileDataset} from './utils/vector-tile-utils';
 import TileDataset from './common-tile/tile-dataset';
 import {isIndexedField, isDomainQuantiles} from './common-tile/tile-utils';
 

src/layers/src/vector-tile/common-tile/tile-dataset.ts

@@ -5,7 +5,7 @@ import {Field as KeplerField} from '@kepler.gl/types';
 import {quickInsertionSort} from '@kepler.gl/utils';
 
 import IterableTileSet, {RowCountAccessor} from './iterable-tile-set';
-import {pruneQuantiles} from '../utils/vector-tile-utils';
+import {pruneQuantiles} from './tile-utils';
 
 export type Datum = number | string | null;
 

src/layers/src/vector-tile/common-tile/tile-utils.ts

@@ -48,3 +48,16 @@ export function getPropertyByZoom(
     return stops[i - 1][1];
   };
 }
+
+/**
+ * Remove null/0 values from the bottom of the quantiles. If the column has many nulls
+ * or 0s at the bottom of the quantiles, it will wash out color scales and produce
+ * meaningless "no value" legend entries. We want to keep the first 0 and no others.
+ * Operates in place.
+ */
+export function pruneQuantiles(quantiles: number[]): void {
+  const firstNonZeroIdx = quantiles.findIndex(d => d !== null && d !== 0);
+  if (firstNonZeroIdx > 0) {
+    quantiles.splice(0, firstNonZeroIdx - 1);
+  }
+}

src/layers/src/vector-tile/vector-tile-layer.ts

@@ -12,15 +12,22 @@ import GL from '@luma.gl/constants';
 
 import {notNullorUndefined} from '@kepler.gl/common-utils';
 import {
+  DatasetType,
+  TileType,
+  VectorTileType,
+  VectorTileDatasetMetadata,
   SCALE_TYPES,
   CHANNEL_SCALES,
   DEFAULT_COLOR_UI,
   LAYER_VIS_CONFIGS
 } from '@kepler.gl/constants';
 import {
+  getTileUrl,
+  getLoaderOptions,
   KeplerTable as KeplerDataset,
   Datasets as KeplerDatasets,
-  GpuFilter
+  GpuFilter,
+  VectorTileMetadata
 } from '@kepler.gl/table';
 import {
   AnimationConfig,
@@ -36,7 +43,6 @@ import {
 } from '@kepler.gl/types';
 import {DataContainerInterface} from '@kepler.gl/utils';
 
-import {getTileUrl} from './utils/vector-tile-utils';
 import {MVTLayer as CustomMVTLayer} from './mvt-layer';
 import VectorTileIcon from './vector-tile-icon';
 import {
@@ -47,14 +53,6 @@ import {
   VisualChannelDomain,
   VisualChannelField
 } from '../base-layer';
-import {
-  VectorTileMetadata,
-  getLoaderOptions,
-  DatasetType,
-  TileType,
-  VectorTileType,
-  VectorTileDatasetMetadata
-} from './utils/vector-tile-utils';
 
 import AbstractTileLayer, {
   LayerData as CommonLayerData,

src/table/package.json

@@ -34,13 +34,16 @@
     "@kepler.gl/constants": "3.1.0-alpha.3",
     "@kepler.gl/types": "3.1.0-alpha.3",
     "@kepler.gl/utils": "3.1.0-alpha.3",
+    "@loaders.gl/mvt": "^4.3.2",
+    "@loaders.gl/pmtiles": "^4.3.2",
     "@types/d3-array": "^2.8.0",
     "@types/lodash.uniq": "^4.5.7",
     "d3-array": "^2.8.0",
     "global": "^4.3.0",
     "lodash.uniq": "^4.0.1",
     "moment": "^2.10.6",
-    "react-palm": "^3.3.8"
+    "react-palm": "^3.3.8",
+    "type-analyzer": "0.4.0"
   },
   "nyc": {
     "sourceMap": false,

src/table/src/dataset-utils.ts

@@ -6,12 +6,18 @@ import KeplerTable, {Datasets} from './kepler-table';
 import {ProtoDataset, RGBColor} from '@kepler.gl/types';
 import Task from 'react-palm/tasks';
 
+import {DatasetType, VectorTileType} from '@kepler.gl/constants';
 import {
   hexToRgb,
   validateInputData,
   datasetColorMaker,
   getApplicationConfig
 } from '@kepler.gl/utils';
+import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
+import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
+
+import {getMVTMetadata} from './tileset/tileset-utils';
+import {parseVectorMetadata} from './tileset/vector-tile-utils';
 
 // apply a color for each dataset
 // to use as label colors
@@ -80,12 +86,71 @@ async function updateTable({table, data}) {
   return updated;
 }
 
-async function createTable({info, color, opts, data}) {
+type CreateTableProps = {
+  info: any;
+  color: RGBColor;
+  opts: {
+    metadata?: Record<string, unknown>;
+  };
+  data: any;
+};
+
+async function createTable(dataasetInfo: CreateTableProps) {
+  const {info, color, opts, data} = dataasetInfo;
+
+  // update metadata for remote tiled datasets
+  const refreshedMetadata = await refreshMetadata(dataasetInfo);
+  const metadata = refreshedMetadata ? {...opts.metadata, ...refreshedMetadata} : opts.metadata;
+
   const TableClass = getApplicationConfig().table ?? KeplerTable;
-  const table = new TableClass({info, color, ...opts});
+  const table = new TableClass({
+    info,
+    color,
+    ...opts,
+    metadata
+  });
   await table.importData({data});
 
   return table;
 }
 const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');
 const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');
+
+/**
+ * Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
+ * @param datasetInfo
+ * @returns
+ */
+async function refreshMetadata(datasetInfo: CreateTableProps) {
+  // so far only vector tile layers should refresh metadata
+  if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {
+    return null;
+  }
+
+  const {type, tilesetMetadataUrl} = datasetInfo.opts.metadata || {};
+
+  if (
+    !(type === VectorTileType.PMTILES || type === VectorTileType.MVT) ||
+    typeof tilesetMetadataUrl !== 'string'
+  ) {
+    return null;
+  }
+
+  try {
+    let rawMetadata: PMTilesMetadata | TileJSON | null = null;
+    if (type === VectorTileType.MVT) {
+      rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
+    } else {
+      const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});
+      rawMetadata = await tileSource.metadata;
+    }
+
+    if (rawMetadata) {
+      return parseVectorMetadata(rawMetadata);
+    }
+  } catch (err) {
+    // ignore for now, and use old metadata?
+  }
+
+  return null;
+}

src/table/src/index.ts

@@ -23,3 +23,5 @@ export type {
 } from './kepler-table';
 export * from './gpu-filter-utils';
 export * from './dataset-utils';
+export * from './tileset/tileset-utils';
+export * from './tileset/vector-tile-utils';

src/layers/src/vector-tile/mvt-source-utils.ts → src/table/src/tileset/tileset-utils.ts

@@ -1,12 +1,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-import {
-  MVTSource as _MVTSource,
-  MVTTileSource as _MVTTileSource,
-  TileJSONLoader,
-  TileJSON
-} from '@loaders.gl/mvt';
+import {TileJSONLoader, TileJSON} from '@loaders.gl/mvt';
 
 /**
  * MVTSource in current loaders ignores attribution

src/layers/src/vector-tile/utils/vector-tile-utils.spec.ts → src/table/src/tileset/vector-tile-utils.spec.ts

@@ -6,7 +6,7 @@ import {DATA_TYPES} from 'type-analyzer';
 import {ALL_FIELD_TYPES, FILTER_TYPES} from '@kepler.gl/constants';
 
 import {getTileUrl, getMetaUrl, parseVectorMetadata as parseMetadata} from './vector-tile-utils';
-import {PMTILES_METADATA, MVT_METADATA} from '../../../../../test/fixtures/tile-metadata';
+import {PMTILES_METADATA, MVT_METADATA} from '../../../../test/fixtures/tile-metadata';
 
 describe('getTileUrl', () => {
   [