CARTO: binary data for spatial index layers (visgl#7160)

jeffpeck10x · Aug 10, 2022 · d6b101c · d6b101c
1 parent 87ddf6c
commit d6b101c
Show file tree

Hide file tree

Showing 12 changed files with 406 additions and 42 deletions.
diff --git a/modules/carto/src/layers/carto-tile-layer.ts b/modules/carto/src/layers/carto-tile-layer.ts
@@ -12,7 +12,7 @@ import {
 } from '@deck.gl/geo-layers';
 import {GeoJsonLayer} from '@deck.gl/layers';
 import {geojsonToBinary} from '@loaders.gl/gis';
-import {Properties, Tile, TileReader} from './schema/carto-tile';
+import {KeyValueProperties, Tile, TileReader} from './schema/carto-tile';
 import {TileFormat, TILE_FORMATS} from '../api/maps-api-common';
 import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {BinaryFeatures} from '@loaders.gl/schema';
@@ -28,7 +28,7 @@ function parsePbf(buffer: ArrayBuffer): Tile {
   return tile;
 }
 
-function unpackProperties(properties: Properties[]) {
+function unpackProperties(properties: KeyValueProperties[]) {
   if (!properties || !properties.length) {
     return [];
   }
@@ -67,7 +67,10 @@ const CartoTileLoader: LoaderWithParser = {
   id: 'cartoTile',
   module: 'carto',
   extensions: ['pbf'],
-  mimeTypes: ['application/x-protobuf'],
+  mimeTypes: [
+    'application/vnd.carto-vector-tile',
+    'application/x-protobuf' // Back-compatibility
+  ],
   category: 'geometry',
   worker: false,
   parse: async (arrayBuffer, options) => parseCartoTile(arrayBuffer, options),

diff --git a/modules/carto/src/layers/quadbin-utils.ts b/modules/carto/src/layers/quadbin-utils.ts
@@ -12,11 +12,11 @@ const B = [
 ];
 const S = [0n, 1n, 2n, 4n, 8n, 16n];
 
-function indexToBigInt(index: string): bigint {
+export function indexToBigInt(index: string): bigint {
   return BigInt(`0x${index}`);
 }
 
-function bigIntToIndex(quadbin: bigint): string {
+export function bigIntToIndex(quadbin: bigint): string {
   return quadbin.toString(16);
 }
 

diff --git a/modules/carto/src/layers/schema/carto-spatial-tile-loader.ts b/modules/carto/src/layers/schema/carto-spatial-tile-loader.ts
@@ -0,0 +1,59 @@
+import Protobuf from 'pbf';
+import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
+
+import {KeyValueProperties} from './carto-tile';
+import {binaryToSpatialjson, Properties, SpatialJson} from './spatialjson-utils';
+import {Tile, TileReader} from './carto-spatial-tile';
+
+const CartoSpatialTileLoader: LoaderWithParser = {
+  name: 'CARTO Spatial Tile',
+  version: '1',
+  id: 'cartoSpatialTile',
+  module: 'carto',
+  extensions: ['pbf'],
+  mimeTypes: [
+    'application/vnd.carto-spatial-tile',
+    'application/x-protobuf' // Back-compatibility
+  ],
+  category: 'geometry',
+  worker: false,
+  parse: async (arrayBuffer, options) => parseCartoSpatialTile(arrayBuffer, options),
+  parseSync: parseCartoSpatialTile,
+  options: {}
+};
+
+function parsePbf(buffer: ArrayBuffer): Tile {
+  const pbf = new Protobuf(buffer);
+  const tile = TileReader.read(pbf);
+  return tile;
+}
+
+function unpackProperties(properties: KeyValueProperties[]): Properties[] {
+  if (!properties || !properties.length) {
+    return [];
+  }
+  return properties.map(item => {
+    const currentRecord: Properties = {};
+    item.data.forEach(({key, value}) => {
+      currentRecord[key] = value;
+    });
+    return currentRecord;
+  });
+}
+
+function parseCartoSpatialTile(
+  arrayBuffer: ArrayBuffer,
+  options?: LoaderOptions
+): SpatialJson | null {
+  if (!arrayBuffer) return null;
+  const tile = parsePbf(arrayBuffer);
+
+  const {cells} = tile;
+  const data = {
+    cells: {...cells, properties: unpackProperties(cells.properties)}
+  };
+
+  return binaryToSpatialjson(data);
+}
+
+export default CartoSpatialTileLoader;
diff --git a/modules/carto/src/layers/schema/carto-spatial-tile.ts b/modules/carto/src/layers/schema/carto-spatial-tile.ts
@@ -0,0 +1,83 @@
+import {Indices, IndexScheme} from './spatialjson-utils';
+import {
+  KeyValueProperties,
+  NumericProp,
+  NumericPropKeyValueReader,
+  PropertiesReader
+} from './carto-tile';
+
+// Indices =====================================
+
+export class IndicesReader {
+  static read(pbf, end?: number): Indices {
+    const {value} = pbf.readFields(IndicesReader._readField, {value: []}, end);
+    return {value: new BigUint64Array(value)};
+  }
+  static _readField(this: void, tag: number, obj, pbf) {
+    if (tag === 1) readPackedFixed64(pbf, obj.value);
+  }
+}
+
+// Cells =========================================
+
+interface Cells {
+  indices: Indices;
+  properties: KeyValueProperties[];
+  numericProps: Record<string, NumericProp>;
+}
+
+class CellsReader {
+  static read(pbf, end?: number): Cells {
+    return pbf.readFields(
+      CellsReader._readField,
+      {indices: null, properties: [], numericProps: {}},
+      end
+    );
+  }
+  static _readField(this: void, tag: number, obj: Cells, pbf) {
+    if (tag === 1) obj.indices = IndicesReader.read(pbf, pbf.readVarint() + pbf.pos);
+    else if (tag === 2) obj.properties.push(PropertiesReader.read(pbf, pbf.readVarint() + pbf.pos));
+    else if (tag === 3) {
+      const entry = NumericPropKeyValueReader.read(pbf, pbf.readVarint() + pbf.pos);
+      obj.numericProps[entry.key] = entry.value;
+    }
+  }
+}
+
+// Tile ========================================
+
+// TODO this type is very similar to SpatialBinary, should align
+export interface Tile {
+  scheme: IndexScheme;
+  cells: Cells;
+}
+
+export class TileReader {
+  static read(pbf, end?: number): Tile {
+    return pbf.readFields(TileReader._readField, {scheme: 0, cells: null}, end);
+  }
+  static _readField(this: void, tag: number, obj: Tile, pbf) {
+    if (tag === 1) obj.scheme = pbf.readVarint();
+    else if (tag === 2) obj.cells = CellsReader.read(pbf, pbf.readVarint() + pbf.pos);
+  }
+}
+
+// pbf doesn't support BigInt natively, implement support for packed fixed64 type
+const SHIFT_LEFT_32 = BigInt((1 << 16) * (1 << 16));
+
+function readPackedEnd(pbf) {
+  return pbf.type === 2 ? pbf.readVarint() + pbf.pos : pbf.pos + 1;
+}
+function readFixed64(pbf) {
+  const a = BigInt(pbf.readFixed32());
+  const b = BigInt(pbf.readFixed32());
+  return a + b * SHIFT_LEFT_32;
+}
+
+function readPackedFixed64(pbf, arr) {
+  if (pbf.type !== 2) return arr.push(readFixed64(pbf));
+  const end = readPackedEnd(pbf);
+  arr = arr || [];
+  while (pbf.pos < end) arr.push(readFixed64(pbf));
+  return arr;
+}
diff --git a/modules/carto/src/layers/schema/carto-tile.ts b/modules/carto/src/layers/schema/carto-tile.ts
@@ -16,15 +16,15 @@ class KeyValueObjectReader {
 
 // Properties ========================================
 
-export interface Properties {
+export interface KeyValueProperties {
   data: KeyValueObject[];
 }
 
-class PropertiesReader {
+export class PropertiesReader {
   static read(pbf, end?: number) {
     return pbf.readFields(PropertiesReader._readField, {data: []}, end);
   }
-  static _readField(this: void, tag: number, obj: Properties, pbf) {
+  static _readField(this: void, tag: number, obj: KeyValueProperties, pbf) {
     if (tag === 1) obj.data.push(KeyValueObjectReader.read(pbf, pbf.readVarint() + pbf.pos));
   }
 }
@@ -69,7 +69,7 @@ class IntsReader {
 
 // NumericProp ========================================
 
-interface NumericProp {
+export interface NumericProp {
   value: number[];
 }
 
@@ -88,7 +88,7 @@ interface NumbericPropKeyValue {
   value: NumericProp;
 }
 
-class NumericPropKeyValueReader {
+export class NumericPropKeyValueReader {
   static read(pbf, end?: number): NumbericPropKeyValue {
     return pbf.readFields(NumericPropKeyValueReader._readField, {key: '', value: null}, end);
   }
@@ -104,7 +104,7 @@ interface Points {
   positions: Doubles;
   globalFeatureIds: Ints;
   featureIds: Ints;
-  properties: Properties[];
+  properties: KeyValueProperties[];
   numericProps: Record<string, NumericProp>;
 }
 

diff --git a/modules/carto/src/layers/schema/spatialjson-utils.ts b/modules/carto/src/layers/schema/spatialjson-utils.ts
@@ -0,0 +1,35 @@
+import {bigIntToIndex} from '../quadbin-utils';
+
+export type IndexScheme = 'h3' | 'quadbin';
+type TypedArray = Float32Array | Float64Array;
+
+export type Indices = {value: BigUint64Array};
+export type NumericProps = Record<string, {value: number[] | TypedArray}>;
+export type Properties = Record<string, string | number | boolean | null>;
+export type Cells = {
+  indices: Indices;
+  numericProps: NumericProps;
+  properties: Properties[];
+};
+export type SpatialBinary = {scheme?: IndexScheme; cells: Cells};
+export type SpatialJson = {
+  id: string;
+  properties: Properties;
+}[];
+
+export function binaryToSpatialjson(binary: SpatialBinary): SpatialJson {
+  const {cells} = binary;
+  const count = cells.indices.value.length;
+  const spatial: any[] = [];
+  for (let i = 0; i < count; i++) {
+    const id = bigIntToIndex(cells.indices.value[i]);
+
+    const properties = {...cells.properties[i]};
+    for (const key of Object.keys(cells.numericProps)) {
+      properties[key] = cells.numericProps[key].value[i];
+    }
+    spatial.push({id, properties});
+  }
+
+  return spatial;
+}
diff --git a/modules/carto/src/layers/spatial-index-tile-layer.ts b/modules/carto/src/layers/spatial-index-tile-layer.ts
@@ -1,3 +1,7 @@
+import {registerLoaders} from '@loaders.gl/core';
+import CartoSpatialTileLoader from './schema/carto-spatial-tile-loader';
+registerLoaders([CartoSpatialTileLoader]);
+
 import {PickingInfo} from '@deck.gl/core';
 import {TileLayer, _Tile2DHeader as Tile2DHeader} from '@deck.gl/geo-layers';