feat(carto): Allow configuring max URL length for GET requests (#9159)

visgl · Sep 23, 2024 · 76ffe2f · 76ffe2f
1 parent feb7bf8
commit 76ffe2f
Show file tree

Hide file tree

Showing 8 changed files with 119 additions and 45 deletions.
diff --git a/docs/api-reference/carto/data-sources.md b/docs/api-reference/carto/data-sources.md
@@ -11,8 +11,8 @@ import {vectorTableSource} from '@deck.gl/carto';
 const data = vectorTableSource({
   accessToken: 'XXX',
   connectionName: 'carto_dw',
-  tableName: 'carto-demo-data.demo_tables.chicago_crime_sample',
-})
+  tableName: 'carto-demo-data.demo_tables.chicago_crime_sample'
+});
 ```
 
 ### Promise API
@@ -51,7 +51,7 @@ type SourceOptions = {
   apiBaseUrl?: string;
   clientId?: string;
   headers?: Record<string, string>;
-  mapsUrl?: string;
+  maxLengthURL?: number;
 };
 ```
 
@@ -64,7 +64,7 @@ type VectorTableSourceOptions = {
   columns?: string[];
   spatialDataColumn?: string;
   tableName: string;
-}
+};
 ```
 
 #### vectorQuerySource
@@ -74,15 +74,15 @@ type VectorQuerySourceOptions = {
   spatialDataColumn?: string;
   sqlQuery: string;
   queryParameters: QueryParameters;
-}
+};
 ```
 
 #### vectorTilesetSource
 
 ```ts
 type VectorTilesetSourceOptions = {
   tableName: string;
-}
+};
 ```
 
 #### h3TableSource
@@ -94,7 +94,7 @@ type H3TableSourceOptions = {
   columns?: string[];
   spatialDataColumn?: string;
   tableName: string;
-}
+};
 ```
 
 #### h3QuerySource
@@ -106,15 +106,15 @@ type H3QuerySourceOptions = {
   spatialDataColumn?: string;
   sqlQuery: string;
   queryParameters: QueryParameters;
-}
+};
 ```
 
 #### h3TilesetSource
 
 ```ts
 type H3TilesetSourceOptions = {
   tableName: string;
-}
+};
 ```
 
 #### quadbinTableSource
@@ -126,7 +126,7 @@ type QuadbinTableSourceOptions = {
   columns?: string[];
   spatialDataColumn?: string;
   tableName: string;
-}
+};
 ```
 
 #### quadbinQuerySource
@@ -138,23 +138,23 @@ type QuadbinQuerySourceOptions = {
   spatialDataColumn?: string;
   sqlQuery: string;
   queryParameters: QueryParameters;
-}
+};
 ```
 
 #### quadbinTilesetSource
 
 ```ts
 type QuadbinTilesetSourceOptions = {
   tableName: string;
-}
+};
 ```
 
 #### rasterTilesetSource (Experimental)
 
 ```ts
 type RasterTilesetSourceOptions = {
   tableName: string;
-}
+};
 ```
 
 Boundary sources are experimental sources where both the tileset and the properties props need a specific schema to work. [Read more about Boundaries in the CARTO documentation](https://docs.carto.com/carto-for-developers/guides/use-boundaries-in-your-application).
@@ -166,7 +166,7 @@ type BoundaryTableSourceOptions = {
   tilesetTableName: string;
   columns?: string[];
   propertiesTableName: string;
-}
+};
 ```
 
 #### boundaryQuerySource (Experimental)
@@ -176,14 +176,15 @@ type BoundaryQuerySourceOptions = {
   tilesetTableName: string;
   propertiesSqlQuery: string;
   queryParameters?: QueryParameters;
-}
+};
 ```
 
 ### QueryParameters
 
 QueryParameters are used to parametrize SQL queries. The format depends on the source's provider, some examples:
 
 [PostgreSQL and Redshift](https://node-postgres.com/features/queries):
+
 ```ts
 vectorQuerySource({
   ...,
@@ -193,6 +194,7 @@ vectorQuerySource({
 ```
 
 [BigQuery positional](https://cloud.google.com/bigquery/docs/parameterized-queries#node.js):
+
 ```ts
 vectorQuerySource({
   ...,
@@ -201,8 +203,8 @@ vectorQuerySource({
 })
 ```
 
-
 [BigQuery named parameters](https://cloud.google.com/bigquery/docs/parameterized-queries#node.js):
+
 ```ts
 vectorQuerySource({
   ...,
@@ -212,6 +214,7 @@ vectorQuerySource({
 ```
 
 [Snowflake positional](https://docs.snowflake.com/en/user-guide/nodejs-driver-use.html#binding-statement-parameters) :
+
 ```ts
 vectorQuerySource({
   ...,
@@ -230,6 +233,7 @@ vectorQuerySource({
 ```
 
 [Databricks ODBC](https://github.com/markdirish/node-odbc#bindparameters-callback)
+
 ```ts
 vectorQuerySource({
   ...

diff --git a/modules/carto/src/api/common.ts b/modules/carto/src/api/common.ts
@@ -1,4 +1,6 @@
 export const DEFAULT_API_BASE_URL = 'https://gcp-us-east1.api.carto.com';
 export const DEFAULT_CLIENT = 'deck-gl-carto';
 export const V3_MINOR_VERSION = '3.4';
-export const MAX_GET_LENGTH = 8192;
+
+// Fastly default limit is 8192; leave some padding.
+export const DEFAULT_MAX_LENGTH_URL = 7000;
diff --git a/modules/carto/src/api/fetch-map.ts b/modules/carto/src/api/fetch-map.ts
@@ -1,6 +1,6 @@
 /* eslint-disable camelcase */
 import {CartoAPIError} from './carto-api-error';
-import {DEFAULT_API_BASE_URL, DEFAULT_CLIENT} from './common';
+import {DEFAULT_API_BASE_URL, DEFAULT_CLIENT, DEFAULT_MAX_LENGTH_URL} from './common';
 import {buildPublicMapUrl, buildStatsUrl} from './endpoints';
 import {
   GeojsonResult,
@@ -36,13 +36,14 @@ type Dataset = {
 };
 
 /* global clearInterval, setInterval, URL */
-/* eslint-disable complexity, max-statements */
+/* eslint-disable complexity, max-statements, max-params */
 async function _fetchMapDataset(
   dataset: Dataset,
   accessToken: string,
   apiBaseUrl: string,
   clientId?: string,
-  headers?: Record<string, string>
+  headers?: Record<string, string>,
+  maxLengthURL = DEFAULT_MAX_LENGTH_URL
 ) {
   const {
     aggregationExp,
@@ -64,7 +65,8 @@ async function _fetchMapDataset(
     clientId,
     connectionName,
     format,
-    headers
+    headers,
+    maxLengthURL
   };
 
   if (type === 'tileset') {
@@ -114,7 +116,8 @@ async function _fetchTilestats(
   attribute: string,
   dataset: Dataset,
   accessToken: string,
-  apiBaseUrl: string
+  apiBaseUrl: string,
+  maxLengthURL = DEFAULT_MAX_LENGTH_URL
 ) {
   const {connectionName, data, id, source, type, queryParameters} = dataset;
   const errorContext: APIErrorContext = {
@@ -144,7 +147,8 @@ async function _fetchTilestats(
     baseUrl,
     headers,
     parameters,
-    errorContext
+    errorContext,
+    maxLengthURL
   });
 
   // Replace tilestats for attribute with value from API
@@ -158,17 +162,19 @@ async function fillInMapDatasets(
   {datasets, token}: {datasets: Dataset[]; token: string},
   clientId: string,
   apiBaseUrl: string,
-  headers?: Record<string, string>
+  headers?: Record<string, string>,
+  maxLengthURL = DEFAULT_MAX_LENGTH_URL
 ) {
   const promises = datasets.map(dataset =>
-    _fetchMapDataset(dataset, token, apiBaseUrl, clientId, headers)
+    _fetchMapDataset(dataset, token, apiBaseUrl, clientId, headers, maxLengthURL)
   );
   return await Promise.all(promises);
 }
 
 async function fillInTileStats(
   {datasets, keplerMapConfig, token}: {datasets: Dataset[]; keplerMapConfig: any; token: string},
-  apiBaseUrl: string
+  apiBaseUrl: string,
+  maxLengthURL = DEFAULT_MAX_LENGTH_URL
 ) {
   const attributes: {attribute: string; dataset: any}[] = [];
   const {layers} = keplerMapConfig.config.visState;
@@ -197,7 +203,7 @@ async function fillInTileStats(
   }
 
   const promises = filteredAttributes.map(({attribute, dataset}) =>
-    _fetchTilestats(attribute, dataset, token, apiBaseUrl)
+    _fetchTilestats(attribute, dataset, token, apiBaseUrl, maxLengthURL)
   );
   return await Promise.all(promises);
 }
@@ -237,6 +243,13 @@ export type FetchMapOptions = {
    * Callback function that will be invoked whenever data in layers is changed. If provided, `autoRefresh` must also be provided.
    */
   onNewData?: (map: any) => void;
+
+  /**
+   * Maximum URL character length. Above this limit, requests use POST.
+   * Used to avoid browser and CDN limits.
+   * @default {@link DEFAULT_MAX_LENGTH_URL}
+   */
+  maxLengthURL?: number;
 };
 
 export type FetchMapResult = ParseMapResult & {
@@ -255,7 +268,8 @@ export async function fetchMap({
   clientId = DEFAULT_CLIENT,
   headers = {},
   autoRefresh,
-  onNewData
+  onNewData,
+  maxLengthURL = DEFAULT_MAX_LENGTH_URL
 }: FetchMapOptions): Promise<FetchMapResult> {
   assert(cartoMapId, 'Must define CARTO map id: fetchMap({cartoMapId: "XXXX-XXXX-XXXX"})');
   assert(apiBaseUrl, 'Must define apiBaseUrl');
@@ -275,18 +289,24 @@ export async function fetchMap({
 
   const baseUrl = buildPublicMapUrl({apiBaseUrl, cartoMapId});
   const errorContext: APIErrorContext = {requestType: 'Public map', mapId: cartoMapId};
-  const map = await requestWithParameters({baseUrl, headers, errorContext});
+  const map = await requestWithParameters({baseUrl, headers, errorContext, maxLengthURL});
 
   // Periodically check if the data has changed. Note that this
   // will not update when a map is published.
   let stopAutoRefresh: (() => void) | undefined;
   if (autoRefresh) {
     // eslint-disable-next-line @typescript-eslint/no-misused-promises
     const intervalId = setInterval(async () => {
-      const changed = await fillInMapDatasets(map, clientId, apiBaseUrl, {
-        ...headers,
-        'If-Modified-Since': new Date().toUTCString()
-      });
+      const changed = await fillInMapDatasets(
+        map,
+        clientId,
+        apiBaseUrl,
+        {
+          ...headers,
+          'If-Modified-Since': new Date().toUTCString()
+        },
+        maxLengthURL
+      );
       if (onNewData && changed.some(v => v === true)) {
         onNewData(parseMap(map));
       }
@@ -315,11 +335,11 @@ export async function fetchMap({
     fetchBasemapProps({config: map.keplerMapConfig.config, errorContext}),
 
     // Mutates map.datasets so that dataset.data contains data
-    fillInMapDatasets(map, clientId, apiBaseUrl, headers)
+    fillInMapDatasets(map, clientId, apiBaseUrl, headers, maxLengthURL)
   ]);
 
   // Mutates attributes in visualChannels to contain tile stats
-  await fillInTileStats(map, apiBaseUrl);
+  await fillInTileStats(map, apiBaseUrl, maxLengthURL);
 
   const out = {...parseMap(map), basemap, ...{stopAutoRefresh}};
 

diff --git a/modules/carto/src/api/query.ts b/modules/carto/src/api/query.ts
@@ -11,6 +11,7 @@ export const query = async function (options: QueryOptions): Promise<QueryResult
   const {
     apiBaseUrl = SOURCE_DEFAULTS.apiBaseUrl,
     clientId = SOURCE_DEFAULTS.clientId,
+    maxLengthURL = SOURCE_DEFAULTS.maxLengthURL,
     connectionName,
     sqlQuery,
     queryParameters
@@ -35,6 +36,7 @@ export const query = async function (options: QueryOptions): Promise<QueryResult
     baseUrl,
     parameters,
     headers,
-    errorContext
+    errorContext,
+    maxLengthURL
   });
 };
diff --git a/modules/carto/src/api/request-with-parameters.ts b/modules/carto/src/api/request-with-parameters.ts
@@ -1,7 +1,7 @@
 import {VERSION} from '@deck.gl/core';
 import {isPureObject} from '../utils';
 import {CartoAPIError} from './carto-api-error';
-import {MAX_GET_LENGTH, V3_MINOR_VERSION} from './common';
+import {DEFAULT_MAX_LENGTH_URL, V3_MINOR_VERSION} from './common';
 import type {APIErrorContext} from './types';
 
 /**
@@ -25,12 +25,14 @@ export async function requestWithParameters<T = any>({
   baseUrl,
   parameters = {},
   headers: customHeaders = {},
-  errorContext
+  errorContext,
+  maxLengthURL = DEFAULT_MAX_LENGTH_URL
 }: {
   baseUrl: string;
   parameters?: Record<string, unknown>;
   headers?: Record<string, string>;
   errorContext: APIErrorContext;
+  maxLengthURL?: number;
 }): Promise<T> {
   parameters = {...DEFAULT_PARAMETERS, ...parameters};
   baseUrl = excludeURLParameters(baseUrl, Object.keys(parameters));
@@ -44,7 +46,7 @@ export async function requestWithParameters<T = any>({
 
   /* global fetch */
   const fetchPromise =
-    url.length > MAX_GET_LENGTH
+    url.length > maxLengthURL
       ? fetch(baseUrl, {method: 'POST', body: JSON.stringify(parameters), headers})
       : fetch(url, {headers});