CARTO: fetchMap support spatial index layers (#7065)

visgl · Jul 18, 2022 · 779f56a · 779f56a
1 parent 6708cd1
commit 779f56a
Show file tree

Hide file tree

Showing 9 changed files with 485 additions and 108 deletions.
diff --git a/modules/carto/src/api/layer-map.ts b/modules/carto/src/api/layer-map.ts
@@ -19,6 +19,8 @@ import {GeoJsonLayer} from '@deck.gl/layers';
 import {H3HexagonLayer, MVTLayer} from '@deck.gl/geo-layers';
 
 import CartoTileLayer from '../layers/carto-tile-layer';
+import H3TileLayer from '../layers/h3-tile-layer';
+import QuadbinTileLayer from '../layers/quadbin-tile-layer';
 import {TILE_FORMATS} from './maps-api-common';
 import {assert} from '../utils';
 
@@ -111,7 +113,7 @@ export function getLayer(
   config,
   dataset
 ): {Layer: ConstructorOf<Layer>; propMap: any; defaultProps: any} {
-  if (type === 'mvt' || type === 'tileset') {
+  if (type === 'mvt' || type === 'tileset' || type === 'h3' || type === 'quadbin') {
     return getTileLayer(dataset);
   }
 
@@ -163,31 +165,58 @@ export function getLayer(
   };
 }
 
+export function layerFromTileDataset(
+  formatTiles: string | null = TILE_FORMATS.MVT,
+  scheme: string
+): typeof CartoTileLayer | typeof H3TileLayer | typeof MVTLayer | typeof QuadbinTileLayer {
+  if (scheme === 'h3') {
+    return H3TileLayer;
+  }
+  if (scheme === 'quadbin') {
+    return QuadbinTileLayer;
+  }
+  if (formatTiles === TILE_FORMATS.MVT) {
+    return MVTLayer;
+  }
+
+  // formatTiles === BINARY|JSON|GEOJSON
+  return CartoTileLayer;
+}
+
 function getTileLayer(dataset) {
   const {
+    aggregationExp,
+    aggregationResLevel,
     data: {
+      scheme,
       tiles: [tileUrl]
     }
   } = dataset;
   /* global URL */
-  const formatTiles = new URL(tileUrl).searchParams.get('formatTiles') || TILE_FORMATS.MVT;
+  const formatTiles = new URL(tileUrl).searchParams.get('formatTiles');
 
   return {
-    Layer: formatTiles === TILE_FORMATS.MVT ? MVTLayer : CartoTileLayer,
+    Layer: layerFromTileDataset(formatTiles, scheme),
     propMap: sharedPropMap,
     defaultProps: {
       ...defaultProps,
-      uniqueIdProperty: 'geoid',
-      formatTiles
+      ...(aggregationExp && {aggregationExp}),
+      ...(aggregationResLevel && {aggregationResLevel}),
+      formatTiles,
+      uniqueIdProperty: 'geoid'
     }
   };
 }
 
-function domainFromAttribute(attribute, scaleType: SCALE_TYPE) {
+function domainFromAttribute(attribute, scaleType: SCALE_TYPE, scaleLength: number) {
   if (scaleType === 'ordinal' || scaleType === 'point') {
     return attribute.categories.map(c => c.category).filter(c => c !== undefined && c !== null);
   }
 
+  if (scaleType === 'quantile' && attribute.quantiles) {
+    return attribute.quantiles[scaleLength];
+  }
+
   let {min} = attribute;
   if (scaleType === 'log' && min === 0) {
     min = 1e-5;
@@ -211,12 +240,12 @@ function domainFromValues(values, scaleType: SCALE_TYPE) {
   return extent(values);
 }
 
-function calculateDomain(data, name, scaleType) {
+function calculateDomain(data, name, scaleType, scaleLength?) {
   if (data.tilestats) {
     // Tileset data type
     const {attributes} = data.tilestats.layers[0];
     const attribute = attributes.find(a => a.attribute === name);
-    return domainFromAttribute(attribute, scaleType);
+    return domainFromAttribute(attribute, scaleType, scaleLength);
   } else if (data.features) {
     // GeoJSON data type
     const values = data.features.map(({properties}) => properties[name]);
@@ -243,6 +272,25 @@ export function opacityToAlpha(opacity) {
   return opacity !== undefined ? Math.round(255 * Math.pow(opacity, 1 / 2.2)) : 255;
 }
 
+function getAccessorKeys(name: string, aggregation: string | undefined): string[] {
+  let keys = [name];
+  if (aggregation) {
+    // Snowflake will capitalized the keys, need to check lower and upper case version
+    keys = keys.concat([aggregation, aggregation.toUpperCase()].map(a => `${name}_${a}`));
+  }
+  return keys;
+}
+
+function findAccessorKey(keys: string[], properties): string[] {
+  for (const key of keys) {
+    if (key in properties) {
+      return [key];
+    }
+  }
+
+  throw new Error(`Could not find property for any accessor key: ${keys}`);
+}
+
 export function getColorValueAccessor({name}, colorAggregation, data: any) {
   const aggregator = AGGREGATION_FUNC[colorAggregation];
   const accessor = values => aggregator(values, p => p[name]);
@@ -252,7 +300,7 @@ export function getColorValueAccessor({name}, colorAggregation, data: any) {
 export function getColorAccessor(
   {name},
   scaleType: SCALE_TYPE,
-  {colors, colorMap},
+  {aggregation, range: {colors, colorMap}},
   opacity: number | undefined,
   data: any
 ) {
@@ -266,7 +314,7 @@ export function getColorAccessor(
       scaleColor.push(color);
     });
   } else {
-    domain = calculateDomain(data, name, scaleType);
+    domain = calculateDomain(data, name, scaleType, colors.length);
     scaleColor = colors;
   }
 
@@ -279,21 +327,36 @@ export function getColorAccessor(
   scale.unknown(UNKNOWN_COLOR);
   const alpha = opacityToAlpha(opacity);
 
+  let accessorKeys = getAccessorKeys(name, aggregation);
   const accessor = properties => {
-    const propertyValue = properties[name];
+    if (!(accessorKeys[0] in properties)) {
+      accessorKeys = findAccessorKey(accessorKeys, properties);
+    }
+    const propertyValue = properties[accessorKeys[0]];
     const {r, g, b} = rgb(scale(propertyValue));
     return [r, g, b, propertyValue === null ? 0 : alpha];
   };
   return normalizeAccessor(accessor, data);
 }
 
-export function getSizeAccessor({name}, scaleType: SCALE_TYPE, range: Iterable<Range>, data: any) {
+export function getSizeAccessor(
+  {name},
+  scaleType: SCALE_TYPE,
+  aggregation,
+  range: Iterable<Range>,
+  data: any
+) {
   const scale = SCALE_FUNCS[scaleType as any]();
   scale.domain(calculateDomain(data, name, scaleType));
   scale.range(range);
 
+  let accessorKeys = getAccessorKeys(name, aggregation);
   const accessor = properties => {
-    return scale(properties[name]);
+    if (!(accessorKeys[0] in properties)) {
+      accessorKeys = findAccessorKey(accessorKeys, properties);
+    }
+    const propertyValue = properties[accessorKeys[0]];
+    return scale(propertyValue);
   };
   return normalizeAccessor(accessor, data);
 }

diff --git a/modules/carto/src/api/maps-v3-client.ts b/modules/carto/src/api/maps-v3-client.ts
@@ -1,7 +1,12 @@
 /**
  * Maps API Client for Carto 3
  */
-import {getDefaultCredentials, buildMapsUrlFromBase, CloudNativeCredentials} from '../config';
+import {
+  getDefaultCredentials,
+  buildMapsUrlFromBase,
+  buildStatsUrlFromBase,
+  CloudNativeCredentials
+} from '../config';
 import {
   API_VERSIONS,
   COLUMNS_SUPPORT,
@@ -144,6 +149,9 @@ function getParameters({
   }
   if (aggregationExp) {
     parameters.push(encodeParameter('aggregationExp', aggregationExp));
+  } else if (isSpatialIndexGeoColumn(geoColumn)) {
+    // Default aggregationExp required for spatial index layers
+    parameters.push(encodeParameter('aggregationExp', '1 AS value'));
   }
   if (aggregationResLevel) {
     parameters.push(encodeParameter('aggregationResLevel', aggregationResLevel));
@@ -152,6 +160,11 @@ function getParameters({
   return parameters.join('&');
 }
 
+function isSpatialIndexGeoColumn(geoColumn: string | undefined) {
+  const spatialIndex = geoColumn?.split(':')[0];
+  return spatialIndex === 'h3' || spatialIndex === 'quadbin';
+}
+
 export async function mapInstantiation({
   type,
   source,
@@ -170,8 +183,8 @@ export async function mapInstantiation({
     geoColumn,
     columns,
     clientId,
-    aggregationExp,
-    aggregationResLevel
+    aggregationResLevel,
+    aggregationExp
   })}`;
   const {accessToken} = credentials;
 
@@ -363,9 +376,20 @@ async function _fetchMapDataset(
   credentials: CloudNativeCredentials,
   clientId?: string
 ) {
-  const {connectionName: connection, columns, format, geoColumn, source, type} = dataset;
+  const {
+    aggregationExp,
+    aggregationResLevel,
+    connectionName: connection,
+    columns,
+    format,
+    geoColumn,
+    source,
+    type
+  } = dataset;
   // First fetch metadata
   const {url, mapFormat} = await _fetchDataUrl({
+    aggregationExp,
+    aggregationResLevel,
     clientId,
     credentials: {...credentials, accessToken},
     connection,
@@ -389,6 +413,31 @@ async function _fetchMapDataset(
   return true;
 }
 
+async function _fetchTilestats(
+  attribute,
+  dataset,
+  accessToken: string,
+  credentials: CloudNativeCredentials
+) {
+  const {connectionName: connection, source, type} = dataset;
+
+  const statsUrl = buildStatsUrlFromBase(credentials.apiBaseUrl);
+  let url = `${statsUrl}/${connection}/`;
+  if (type === MAP_TYPES.QUERY) {
+    url += `${attribute}?q=${source}`;
+  } else {
+    // MAP_TYPE.TABLE
+    url += `${source}/${attribute}`;
+  }
+  const stats = await requestData({url, format: FORMATS.JSON, accessToken});
+
+  // Replace tilestats for attribute with value from API
+  const {attributes} = dataset.data.tilestats.layers[0];
+  const index = attributes.findIndex(d => d.attribute === attribute);
+  attributes[index] = stats;
+  return true;
+}
+
 async function fillInMapDatasets(
   {datasets, token},
   clientId: string,
@@ -398,6 +447,42 @@ async function fillInMapDatasets(
   return await Promise.all(promises);
 }
 
+async function fillInTileStats(
+  {datasets, keplerMapConfig, token},
+  credentials: CloudNativeCredentials
+) {
+  const attributes: {attribute?: string; dataset?: any}[] = [];
+  const {layers} = keplerMapConfig.config.visState;
+  for (const layer of layers) {
+    for (const channel of Object.keys(layer.visualChannels)) {
+      const attribute = layer.visualChannels[channel]?.name;
+      if (attribute) {
+        const dataset = datasets.find(d => d.id === layer.config.dataId);
+        if (dataset.data.tilestats && dataset.type !== MAP_TYPES.TILESET) {
+          // Only fetch stats for QUERY & TABLE map types
+          attributes.push({attribute, dataset});
+        }
+      }
+    }
+  }
+  // Remove duplicates to avoid repeated requests
+  const filteredAttributes: {attribute?: string; dataset?: any}[] = [];
+  for (const a of attributes) {
+    if (
+      !filteredAttributes.find(
+        ({attribute, dataset}) => attribute === a.attribute && dataset === a.dataset
+      )
+    ) {
+      filteredAttributes.push(a);
+    }
+  }
+
+  const promises = filteredAttributes.map(({attribute, dataset}) =>
+    _fetchTilestats(attribute, dataset, token, credentials)
+  );
+  return await Promise.all(promises);
+}
+
 export async function fetchMap({
   cartoMapId,
   clientId,
@@ -466,6 +551,9 @@ export async function fetchMap({
 
   // Mutates map.datasets so that dataset.data contains data
   await fillInMapDatasets(map, clientId, localCreds);
+
+  // Mutates attributes in visualChannels to contain tile stats
+  await fillInTileStats(map, localCreds);
   return {
     ...parseMap(map),
     ...{stopAutoRefresh}

diff --git a/modules/carto/src/api/parseMap.ts b/modules/carto/src/api/parseMap.ts
@@ -176,10 +176,11 @@ function createChannelProps(visualChannels, type, config, data) {
       }
     }
   } else if (colorField) {
+    const {colorAggregation: aggregation, colorRange: range} = visConfig;
     result.getFillColor = getColorAccessor(
       colorField,
       colorScale,
-      visConfig.colorRange,
+      {aggregation, range},
       visConfig.opacity,
       data
     );
@@ -189,10 +190,11 @@ function createChannelProps(visualChannels, type, config, data) {
     const fallbackOpacity = type === 'point' ? visConfig.opacity : 1;
     const opacity =
       visConfig.strokeOpacity !== undefined ? visConfig.strokeOpacity : fallbackOpacity;
+    const {strokeColorAggregation: aggregation, strokeColorRange: range} = visConfig;
     result.getLineColor = getColorAccessor(
       strokeColorField,
       strokeColorScale,
-      visConfig.strokeColorRange,
+      {aggregation, range},
       opacity,
       data
     );
@@ -201,6 +203,7 @@ function createChannelProps(visualChannels, type, config, data) {
     result.getElevation = getSizeAccessor(
       heightField,
       heightScale,
+      visConfig.heightAggregation,
       visConfig.heightRange || visConfig.sizeRange,
       data
     );
@@ -209,6 +212,7 @@ function createChannelProps(visualChannels, type, config, data) {
     result.getPointRadius = getSizeAccessor(
       sizeField,
       sizeScale,
+      visConfig.sizeAggregation,
       visConfig.radiusRange || visConfig.sizeRange,
       data
     );