diff --git a/modules/shapefile/src/dbf-loader.ts b/modules/shapefile/src/dbf-loader.ts index 9ef2beda90..2efbaf65b9 100644 --- a/modules/shapefile/src/dbf-loader.ts +++ b/modules/shapefile/src/dbf-loader.ts @@ -1,14 +1,29 @@ -import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils'; +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + +import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils'; +import type {ObjectRowTable} from '@loaders.gl/schema'; +// import type {DBFResult} from './lib/parsers/parse-dbf'; import {parseDBF, parseDBFInBatches} from './lib/parsers/parse-dbf'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest'; +/** Options for loading DBF files */ +export type DBFLoaderOptions = LoaderOptions & { + dbf?: { + /** Shape of returned table */ + shape?: 'object-row-table'; + /** Encoding of strings in table */ + encoding?: string; + }; +}; + /** * DBFLoader - DBF files are used to contain non-geometry columns in Shapefiles */ -export const DBFWorkerLoader: Loader = { +export const DBFWorkerLoader: Loader = { name: 'DBF', id: 'dbf', module: 'shapefile', @@ -19,17 +34,49 @@ export const DBFWorkerLoader: Loader = { mimeTypes: ['application/x-dbf'], options: { dbf: { + shape: 'object-row-table', encoding: 'latin1' } } }; /** DBF file loader */ -export const DBFLoader: LoaderWithParser = { +export const DBFLoader: LoaderWithParser = { ...DBFWorkerLoader, +<<<<<<< Updated upstream + parse: async (arrayBuffer: ArrayBuffer, options?: DBFLoaderOptions) => { + const dbfOptions = {...DBFLoader.options, ...options?.dbf}; + return parseDBF(arrayBuffer, dbfOptions); + }. + parseSync: parseDBF, +======= +<<<<<<< Updated upstream parse: async (arrayBuffer, options) => parseDBF(arrayBuffer, options), parseSync: parseDBF, parseInBatches(arrayBufferIterator: AsyncIterable | Iterable, options) { return parseDBFInBatches(arrayBufferIterator, options); +======= + + parse: async (arrayBuffer: ArrayBuffer, options?: DBFLoaderOptions) => { + const dbfOptions = {...DBFLoader.options, ...options?.dbf}; + return parseDBF(arrayBuffer, dbfOptions); + }, + + parseSync: (arrayBuffer: ArrayBuffer, options?: DBFLoaderOptions) => { + const dbfOptions = {...DBFLoader.options, ...options?.dbf}; + return parseDBF(arrayBuffer, dbfOptions); + }, + +>>>>>>> Stashed changes + parseInBatches( + arrayBufferIterator: AsyncIterable | Iterable, + options?: DBFLoaderOptions + ): AsyncIterableIterator { + const dbfOptions = {...DBFLoader.options, ...options?.dbf}; + return parseDBFInBatches(arrayBufferIterator, dbfOptions); +<<<<<<< Updated upstream +======= +>>>>>>> Stashed changes +>>>>>>> Stashed changes } }; diff --git a/modules/shapefile/src/index.ts b/modules/shapefile/src/index.ts index 6f0504156c..96d7d534d7 100644 --- a/modules/shapefile/src/index.ts +++ b/modules/shapefile/src/index.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + export {ShapefileLoader} from './shapefile-loader'; export {DBFLoader, DBFWorkerLoader} from './dbf-loader'; export {SHPLoader, SHPWorkerLoader} from './shp-loader'; diff --git a/modules/shapefile/src/lib/parsers/parse-dbf.ts b/modules/shapefile/src/lib/parsers/parse-dbf.ts index 915f447a1d..68f73a4c0f 100644 --- a/modules/shapefile/src/lib/parsers/parse-dbf.ts +++ b/modules/shapefile/src/lib/parsers/parse-dbf.ts @@ -1,13 +1,53 @@ -import {Field, ObjectRowTable} from '@loaders.gl/schema'; +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + +import type {Field, ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema'; +import {Schema} from '@loaders.gl/schema'; import {BinaryChunkReader} from '../streaming/binary-chunk-reader'; -import { - DBFLoaderOptions, - DBFResult, - DBFTableOutput, - DBFHeader, - DBFRowsOutput, - DBFField -} from './types'; + +type DBFParserOptions = { + shape?: 'object-row-table'; + encoding?: string; +}; + +export type DBFResult = { + data: {[key: string]: unknown[]}[]; + schema?: Schema; + error?: string; + dbfHeader?: DBFHeader; + dbfFields?: DBFField[]; + progress: { + bytesUsed: number; + rowsTotal: number; + rows: number; + }; +}; + +/** Binary header stored in DBF file */ +export type DBFHeader = { + /** Last updated date - year */ + year: number; + /** Last updated date - month */ + month: number; + /** Last updated date - day */ + day: number; + /** Number of records in data file */ + nRecords: number; + /** Length of header in bytes */ + headerLength: number; + /** Length of each record */ + recordLength: number; + /** Not clear if this is usually set */ + languageDriver: number; +}; + +/** Field descriptor */ +export type DBFField = { + name: string; + dataType: string; + fieldLength: number; + decimal: number; +}; const LITTLE_ENDIAN = true; const DBF_HEADER_SIZE = 32; @@ -25,7 +65,12 @@ class DBFParser { textDecoder: TextDecoder; state = STATE.START; result: DBFResult = { - data: [] + data: [], + progress: { + bytesUsed: 0, + rowsTotal: 0, + rows: 0 + } }; constructor(options: {encoding: string}) { @@ -62,43 +107,64 @@ class DBFParser { * @param options * @returns DBFTable or rows */ +<<<<<<< Updated upstream +export function parseDBF(arrayBuffer: ArrayBuffer, options: DBFParserOptions = {}): ObjectRowTable { +======= +<<<<<<< Updated upstream export function parseDBF( arrayBuffer: ArrayBuffer, options: DBFLoaderOptions = {} ): DBFRowsOutput | DBFTableOutput | ObjectRowTable { +>>>>>>> Stashed changes const {encoding = 'latin1'} = options.dbf || {}; +======= +export function parseDBF(arrayBuffer: ArrayBuffer, options: DBFParserOptions = {}): ObjectRowTable { + const {encoding = 'latin1'} = options; +>>>>>>> Stashed changes const dbfParser = new DBFParser({encoding}); dbfParser.write(arrayBuffer); dbfParser.end(); const {data, schema} = dbfParser.result; - const shape = options?.dbf?.shape; + const shape = options?.shape || 'object-row-table'; switch (shape) { case 'object-row-table': { - const table: ObjectRowTable = { - shape: 'object-row-table', - schema, - data - }; + const table: ObjectRowTable = {shape: 'object-row-table', schema, data}; return table; } - case 'table': - return {schema, rows: data}; - case 'rows': default: - return data; + throw new Error(shape); } + const table: ObjectRowTable = { + shape: 'object-row-table', + schema, + data + }; + return table; } + /** * @param asyncIterator * @param options */ export async function* parseDBFInBatches( asyncIterator: AsyncIterable | Iterable, +<<<<<<< Updated upstream + options: DBFParserOptions = {} +): AsyncIterableIterator { + const {encoding = 'latin1'} = option; +======= +<<<<<<< Updated upstream options: DBFLoaderOptions = {} ): AsyncIterable { const {encoding = 'latin1'} = options.dbf || {}; +======= + options: DBFParserOptions = {} +): AsyncIterableIterator { + const {encoding = 'latin1'} = options; +>>>>>>> Stashed changes +>>>>>>> Stashed changes const parser = new DBFParser({encoding}); let headerReturned = false; @@ -106,26 +172,47 @@ export async function* parseDBFInBatches( parser.write(arrayBuffer); if (!headerReturned && parser.result.dbfHeader) { headerReturned = true; - yield parser.result.dbfHeader; + yield { + batchType: 'metadata', + shape: 'object-row-table', + data: [], + length: 0, + // Additional data + dbfHeader: parser.result.dbfHeader + }; } if (parser.result.data.length > 0) { - yield parser.result.data; + const data = parser.result.data; parser.result.data = []; + yield { + batchType: 'data', + shape: 'object-row-table', + data, + length: data.length + }; } } parser.end(); if (parser.result.data.length > 0) { - yield parser.result.data; + const data = parser.result.data; + yield { + batchType: 'data', + shape: 'object-row-table', + data, + length: data.length + }; } } + /** - * https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm + * State machine for DBF parsing * @param state * @param result * @param binaryReader * @param textDecoder * @returns + * @see https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm */ /* eslint-disable complexity, max-depth */ function parseState( @@ -161,8 +248,7 @@ function parseState( case STATE.FIELD_DESCRIPTORS: // Parse DBF field descriptors (schema) const fieldDescriptorView = binaryReader.getDataView( - // @ts-ignore - result.dbfHeader.headerLength - DBF_HEADER_SIZE + result.dbfHeader!.headerLength - DBF_HEADER_SIZE ); if (!fieldDescriptorView) { return state; @@ -191,10 +277,8 @@ function parseState( // Note: Avoid actually reading the last byte, which may not be present binaryReader.skip(1); - // @ts-ignore - const row = parseRow(recordView, result.dbfFields, textDecoder); + const row = parseRow(recordView, result.dbfFields || [], textDecoder); result.data.push(row); - // @ts-ignore result.progress.rows = result.data.length; } state = STATE.END; @@ -218,17 +302,12 @@ function parseState( */ function parseDBFHeader(headerView: DataView): DBFHeader { return { - // Last updated date year: headerView.getUint8(1) + 1900, month: headerView.getUint8(2), day: headerView.getUint8(3), - // Number of records in data file nRecords: headerView.getUint32(4, LITTLE_ENDIAN), - // Length of header in bytes headerLength: headerView.getUint16(8, LITTLE_ENDIAN), - // Length of each record recordLength: headerView.getUint16(10, LITTLE_ENDIAN), - // Not sure if this is usually set languageDriver: headerView.getUint8(29) }; } @@ -266,7 +345,6 @@ function parseRows(binaryReader, fields, nRecords, recordLength, textDecoder) { for (let i = 0; i < nRecords; i++) { const recordView = binaryReader.getDataView(recordLength - 1); binaryReader.skip(1); - // @ts-ignore rows.push(parseRow(recordView, fields, textDecoder)); } return rows; diff --git a/modules/shapefile/src/lib/parsers/parse-shapefile.ts b/modules/shapefile/src/lib/parsers/parse-shapefile.ts index c8d38bf694..36cb77c60c 100644 --- a/modules/shapefile/src/lib/parsers/parse-shapefile.ts +++ b/modules/shapefile/src/lib/parsers/parse-shapefile.ts @@ -1,25 +1,31 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + // import type {Feature} from '@loaders.gl/gis'; -import {LoaderContext, parseInBatchesFromContext, parseFromContext} from '@loaders.gl/loader-utils'; +import type {LoaderContext} from '@loaders.gl/loader-utils'; +import {parseInBatchesFromContext, parseFromContext} from '@loaders.gl/loader-utils'; import {binaryToGeometry, transformGeoJsonCoords} from '@loaders.gl/gis'; -import type {BinaryGeometry, Geometry, ObjectRowTableBatch} from '@loaders.gl/schema'; +import type {ObjectRowTableBatch} from '@loaders.gl/schema'; +import type {BinaryGeometry, GeoJsonProperties, Geometry, Feature} from '@loaders.gl/schema'; import {Proj4Projection} from '@math.gl/proj4'; import type {SHXOutput} from './parse-shx'; import type {SHPHeader} from './parse-shp-header'; -import type {ShapefileLoaderOptions} from './types'; +import type {ShapefileLoaderOptions} from '../../shapefile-loader'; + import {parseShx} from './parse-shx'; import {zipBatchIterators} from '../streaming/zip-batch-iterators'; import {SHPLoader} from '../../shp-loader'; import {DBFLoader} from '../../dbf-loader'; -type Feature = any; -interface ShapefileOutput { +export type ShapefileOutput = { encoding?: string; prj?: string; shx?: SHXOutput; header: SHPHeader; - data: object[]; -} + data: Feature[]; +}; + /** * Parsing of file in batches */ @@ -36,7 +42,7 @@ export async function* parseShapefileInBatches( const shapeIterable = await parseInBatchesFromContext( asyncIterator, SHPLoader, - options, + options || {}, context! ); @@ -71,6 +77,7 @@ export async function* parseShapefileInBatches( let dbfHeader: {batchType?: string} = {}; if (propertyIterator) { + // if (propertyIterable) { dbfHeader = (await propertyIterator.next()).value; if (dbfHeader && dbfHeader.batchType === 'metadata') { dbfHeader = (await propertyIterator.next()).value; @@ -87,11 +94,18 @@ export async function* parseShapefileInBatches( } }; + // let iterator: any; + // if (propertyIterable) { + // iterator = zipBatchIterators(shapeIterator, propertyIterator); + // } else { + // iterator = shapeIterable; + // } + for await (const batch of zippedBatchIterable) { - let geometries: any; - let properties: any; - if (!propertyIterator) { - geometries = batch; + let geometries: BinaryGeometry[]; + let properties: GeoJsonProperties[] = []; + if (!propertyIterable) { + geometries = item; } else { [geometries, properties] = batch.data; } @@ -131,19 +145,21 @@ export async function parseShapefile( // parse geometries const {header, geometries} = await parseFromContext(arrayBuffer, SHPLoader, options, context!); // {shp: shx} + // @ts-expect-error const geojsonGeometries = parseGeometries(geometries); // parse properties - let properties = []; + let properties: any[] = []; const dbfResponse = await context?.fetch(replaceExtension(context?.url!, 'dbf')); if (dbfResponse?.ok) { - properties = await parseFromContext( + const dbf = await parseFromContext( dbfResponse as any, DBFLoader, {dbf: {encoding: cpg || 'latin1'}}, context! ); + properties = dbf.data; } let features = joinProperties(geojsonGeometries, properties); @@ -155,7 +171,7 @@ export async function parseShapefile( encoding: cpg, prj, shx, - header, + header: header!, data: features }; } @@ -167,7 +183,7 @@ export async function parseShapefile( * @returns geometries as an array */ function parseGeometries(geometries: BinaryGeometry[]): Geometry[] { - const geojsonGeometries: any[] = []; + const geojsonGeometries: Geometry[] = []; for (const geom of geometries) { geojsonGeometries.push(binaryToGeometry(geom)); } @@ -181,7 +197,7 @@ function parseGeometries(geometries: BinaryGeometry[]): Geometry[] { * @param properties [description] * @return [description] */ -function joinProperties(geometries: Geometry[], properties: object[]): Feature[] { +function joinProperties(geometries: Geometry[], properties: GeoJsonProperties[]): Feature[] { const features: Feature[] = []; for (let i = 0; i < geometries.length; i++) { const geometry = geometries[i]; diff --git a/modules/shapefile/src/lib/parsers/parse-shp-geometry.ts b/modules/shapefile/src/lib/parsers/parse-shp-geometry.ts index 801fe1a18f..ec040f9567 100644 --- a/modules/shapefile/src/lib/parsers/parse-shp-geometry.ts +++ b/modules/shapefile/src/lib/parsers/parse-shp-geometry.ts @@ -1,5 +1,8 @@ -import {BinaryGeometry, BinaryGeometryType} from '@loaders.gl/schema'; -import {SHPLoaderOptions} from './types'; +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + +import type {BinaryGeometry, BinaryGeometryType} from '@loaders.gl/schema'; +import {SHPLoaderOptions} from '../../shp-loader'; const LITTLE_ENDIAN = true; diff --git a/modules/shapefile/src/lib/parsers/parse-shp-header.ts b/modules/shapefile/src/lib/parsers/parse-shp-header.ts index 3c602ffd39..749749d681 100644 --- a/modules/shapefile/src/lib/parsers/parse-shp-header.ts +++ b/modules/shapefile/src/lib/parsers/parse-shp-header.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + export interface SHPHeader { /** SHP Magic number */ magic: number; diff --git a/modules/shapefile/src/lib/parsers/parse-shp.ts b/modules/shapefile/src/lib/parsers/parse-shp.ts index 94b2ce3712..696ee9dc78 100644 --- a/modules/shapefile/src/lib/parsers/parse-shp.ts +++ b/modules/shapefile/src/lib/parsers/parse-shp.ts @@ -1,9 +1,10 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import type {BinaryGeometry} from '@loaders.gl/schema'; import {BinaryChunkReader} from '../streaming/binary-chunk-reader'; import {parseSHPHeader, SHPHeader} from './parse-shp-header'; import {parseRecord} from './parse-shp-geometry'; -import {SHPLoaderOptions} from './types'; - const LITTLE_ENDIAN = true; const BIG_ENDIAN = false; @@ -19,9 +20,16 @@ const STATE = { ERROR: 3 }; -type SHPResult = { - geometries: (BinaryGeometry | null)[]; +/** + * A complete or partial result from the SHP file parser. + * @note When received as a batch, the `geometries` array will only contain the geometries in that batch. + */ +export type SHPResult = { + /** Metadata from the SHP file header */ header?: SHPHeader; + /** Some rows may have no geometry */ + geometries: (BinaryGeometry | null)[]; + /** Error, if detected */ error?: string; progress: { bytesUsed: number; @@ -31,8 +39,14 @@ type SHPResult = { currentIndex: number; }; +export type SHPParserOptions = { + shp?: { + _maxDimensions?: number; + }; +}; + class SHPParser { - options?: SHPLoaderOptions = {}; + options: SHPParserOptions; binaryReader = new BinaryChunkReader({maxRewindBytes: SHP_RECORD_HEADER_SIZE}); state = STATE.EXPECTING_HEADER; result: SHPResult = { @@ -47,7 +61,7 @@ class SHPParser { currentIndex: NaN }; - constructor(options?: SHPLoaderOptions) { + constructor(options: SHPParserOptions = {}) { this.options = options; } @@ -67,12 +81,12 @@ class SHPParser { } } -export function parseSHP(arrayBuffer: ArrayBuffer, options?: SHPLoaderOptions): BinaryGeometry[] { +export function parseSHP(arrayBuffer: ArrayBuffer, options?: SHPParserOptions): SHPResult { + // BinaryGeometry[] { const shpParser = new SHPParser(options); shpParser.write(arrayBuffer); shpParser.end(); - // @ts-ignore return shpParser.result; } @@ -83,25 +97,37 @@ export function parseSHP(arrayBuffer: ArrayBuffer, options?: SHPLoaderOptions): */ export async function* parseSHPInBatches( asyncIterator: AsyncIterable | Iterable, +<<<<<<< Updated upstream + options?: SHPParserOptions +): AsyncGenerator | AsyncIterable { + +======= +<<<<<<< Updated upstream options?: SHPLoaderOptions ): AsyncGenerator { +======= + options?: SHPParserOptions +): AsyncIterableIterator | AsyncIterableIterator { + +>>>>>>> Stashed changes +>>>>>>> Stashed changes const parser = new SHPParser(options); let headerReturned = false; for await (const arrayBuffer of asyncIterator) { parser.write(arrayBuffer); if (!headerReturned && parser.result.header) { headerReturned = true; - yield parser.result.header; + yield parser.result; } if (parser.result.geometries.length > 0) { - yield parser.result.geometries; + yield parser.result; parser.result.geometries = []; } } parser.end(); if (parser.result.geometries.length > 0) { - yield parser.result.geometries; + yield parser.result; } return; @@ -125,7 +151,7 @@ function parseState( state: number, result: SHPResult, binaryReader: BinaryChunkReader, - options?: SHPLoaderOptions + options?: SHPParserOptions ): number { // eslint-disable-next-line no-constant-condition while (true) { diff --git a/modules/shapefile/src/lib/parsers/parse-shx.ts b/modules/shapefile/src/lib/parsers/parse-shx.ts index 185d80e659..fcf6ae586f 100644 --- a/modules/shapefile/src/lib/parsers/parse-shx.ts +++ b/modules/shapefile/src/lib/parsers/parse-shx.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import {parseSHPHeader} from './parse-shp-header'; export interface SHXOutput { diff --git a/modules/shapefile/src/lib/parsers/types.ts b/modules/shapefile/src/lib/parsers/types.ts deleted file mode 100644 index d6c1cd161a..0000000000 --- a/modules/shapefile/src/lib/parsers/types.ts +++ /dev/null @@ -1,72 +0,0 @@ -import {Schema, ObjectRowTable} from '@loaders.gl/schema'; -import type {LoaderOptions} from '@loaders.gl/loader-utils'; - -export type SHPLoaderOptions = LoaderOptions & { - shp?: { - _maxDimensions?: number; - }; -}; - -export type DBFLoaderOptions = LoaderOptions & { - dbf?: { - encoding?: string; - shape?: 'rows' | 'table' | 'object-row-table'; - }; -}; - -export type ShapefileLoaderOptions = LoaderOptions & - SHPLoaderOptions & { - shapefile?: { - shape?: 'geojson-table'; - }; - gis?: { - reproject?: boolean; - _targetCrs?: string; - }; - }; - -export type DBFRowsOutput = ObjectRowTable['data']; - -/** - * DBF Table output. Deprecated in favor of ObjectRowTable - * @deprecated - */ -export interface DBFTableOutput { - schema?: Schema; - rows: DBFRowsOutput; -} - -export type DBFHeader = { - // Last updated date - year: number; - month: number; - day: number; - // Number of records in data file - nRecords: number; - // Length of header in bytes - headerLength: number; - // Length of each record - recordLength: number; - // Not sure if this is usually set - languageDriver: number; -}; - -export type DBFField = { - name: string; - dataType: string; - fieldLength: number; - decimal: number; -}; - -export type DBFResult = { - data: {[key: string]: any}[]; - schema?: Schema; - error?: string; - dbfHeader?: DBFHeader; - dbfFields?: DBFField[]; - progress?: { - bytesUsed: number; - rowsTotal: number; - rows: number; - }; -}; diff --git a/modules/shapefile/src/lib/streaming/binary-chunk-reader.ts b/modules/shapefile/src/lib/streaming/binary-chunk-reader.ts index c0065e6d80..54e2b2cd26 100644 --- a/modules/shapefile/src/lib/streaming/binary-chunk-reader.ts +++ b/modules/shapefile/src/lib/streaming/binary-chunk-reader.ts @@ -1,7 +1,7 @@ // loaders.gl, MIT license // Copyright (c) vis.gl contributors -export type BinaryChunkReaderOptions = { +export type BinaryChunkReaderProps = { maxRewindBytes: number; }; @@ -11,7 +11,7 @@ export class BinaryChunkReader { ended: boolean; maxRewindBytes: number; - constructor(options?: BinaryChunkReaderOptions) { + constructor(options?: BinaryChunkReaderProps) { const {maxRewindBytes = 0} = options || {}; /** current global offset into current array buffer*/ diff --git a/modules/shapefile/src/lib/streaming/zip-batch-iterators.ts b/modules/shapefile/src/lib/streaming/zip-batch-iterators.ts index 568a56ad89..f9f0a62bf7 100644 --- a/modules/shapefile/src/lib/streaming/zip-batch-iterators.ts +++ b/modules/shapefile/src/lib/streaming/zip-batch-iterators.ts @@ -42,14 +42,22 @@ export async function* zipBatchIterators( } } - const batchData = extractBatchData(batch1Data, batch2Data); - if (batchData) { + // const batchData = extractBatchData(batch1Data, batch2Data); + // if (batchData) { + // yield { + // batchType: 'data', + // shape, + // length: batchData.length, + // data: batchData + // }; + const batch = extractBatchData(batch1.data, batch2.data); + if (batch) { yield { - batchType: 'data', - shape, - length: batchData.length, - data: batchData - }; + batchType: 'data', + shape: 'object-row-table', + length: 0, + data: batch + } } } } @@ -61,14 +69,14 @@ export async function* zipBatchIterators( * @param batch2 * @return array | null */ -function extractBatchData(batch1: any[], batch2: any[]): any[] | null { +function extractBatchData(batch1: unknown[], batch2: unknown[]): unknown[][] | null { const batchLength: number = Math.min(batch1.length, batch2.length); if (batchLength === 0) { return null; } // Non interleaved arrays - const batch: any[] = [batch1.slice(0, batchLength), batch2.slice(0, batchLength)]; + const batch: unknown[][] = [batch1.slice(0, batchLength), batch2.slice(0, batchLength)]; // Modify the 2 batches batch1.splice(0, batchLength); diff --git a/modules/shapefile/src/shapefile-loader.ts b/modules/shapefile/src/shapefile-loader.ts index 24980e2d23..97a69743ac 100644 --- a/modules/shapefile/src/shapefile-loader.ts +++ b/modules/shapefile/src/shapefile-loader.ts @@ -1,16 +1,44 @@ -import type {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils'; +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + +import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils'; +import type {SHPLoaderOptions} from './shp-loader'; +import type {DBFLoaderOptions} from './dbf-loader'; + import {SHP_MAGIC_NUMBER} from './shp-loader'; -import {parseShapefile, parseShapefileInBatches} from './lib/parsers/parse-shapefile'; +import { + parseShapefile, + parseShapefileInBatches, + ShapefileOutput +} from './lib/parsers/parse-shapefile'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest'; +export type ShapefileLoaderOptions = LoaderOptions & + DBFLoaderOptions & + SHPLoaderOptions & { + shapefile?: { + shape?: 'geojson'; + }; + gis?: { + reproject?: boolean; + _targetCrs?: string; + /** @deprecated. Use options.shapefile.shape */ + format?: 'geojson'; + }; + }; + /** * Shapefile loader * @note Shapefile is multifile format and requires providing additional files */ -export const ShapefileLoader: LoaderWithParser = { +export const ShapefileLoader: LoaderWithParser< + ShapefileOutput, + ShapefileOutput, + ShapefileLoaderOptions +> = { name: 'Shapefile', id: 'shapefile', module: 'shapefile', diff --git a/modules/shapefile/src/shp-loader.ts b/modules/shapefile/src/shp-loader.ts index 74a2978514..29ab18e70e 100644 --- a/modules/shapefile/src/shp-loader.ts +++ b/modules/shapefile/src/shp-loader.ts @@ -1,4 +1,8 @@ -import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils'; +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + +import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils'; +import type {SHPResult} from './lib/parsers/parse-shp'; import {parseSHP, parseSHPInBatches} from './lib/parsers/parse-shp'; // __VERSION__ is injected by babel-plugin-version-inline @@ -7,10 +11,16 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest'; export const SHP_MAGIC_NUMBER = [0x00, 0x00, 0x27, 0x0a]; +export type SHPLoaderOptions = LoaderOptions & { + shp?: { + _maxDimensions?: number; + }; +}; + /** * SHP file loader */ -export const SHPWorkerLoader: Loader = { +export const SHPWorkerLoader: Loader = { name: 'SHP', id: 'shp', module: 'shapefile', @@ -29,12 +39,12 @@ export const SHPWorkerLoader: Loader = { }; /** SHP file loader */ -export const SHPLoader: LoaderWithParser = { +export const SHPLoader: LoaderWithParser = { ...SHPWorkerLoader, - parse: async (arrayBuffer, options?) => parseSHP(arrayBuffer, options), - parseSync: parseSHP, + parse: async (arrayBuffer: ArrayBuffer, options?: SHPLoaderOptions) => parseSHP(arrayBuffer, options), + parseSync: (arrayBuffer: ArrayBuffer, options?: SHPLoaderOptions) => parseSHP(arrayBuffer, options), parseInBatches: ( arrayBufferIterator: AsyncIterable | Iterable, - options + options?: SHPLoaderOptions ) => parseSHPInBatches(arrayBufferIterator, options) }; diff --git a/modules/shapefile/src/workers/dbf-worker.ts b/modules/shapefile/src/workers/dbf-worker.ts index 0305f1818a..6204876ce6 100644 --- a/modules/shapefile/src/workers/dbf-worker.ts +++ b/modules/shapefile/src/workers/dbf-worker.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import {DBFLoader} from '../dbf-loader'; import {createLoaderWorker} from '@loaders.gl/loader-utils'; diff --git a/modules/shapefile/src/workers/shp-worker.ts b/modules/shapefile/src/workers/shp-worker.ts index 30d0f1241f..75910b794c 100644 --- a/modules/shapefile/src/workers/shp-worker.ts +++ b/modules/shapefile/src/workers/shp-worker.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import {SHPLoader} from '../shp-loader'; import {createLoaderWorker} from '@loaders.gl/loader-utils'; diff --git a/modules/shapefile/test/dbf-loader.spec.js b/modules/shapefile/test/dbf-loader.spec.ts similarity index 89% rename from modules/shapefile/test/dbf-loader.spec.js rename to modules/shapefile/test/dbf-loader.spec.ts index 53a0c7e0ba..084c00ba97 100644 --- a/modules/shapefile/test/dbf-loader.spec.js +++ b/modules/shapefile/test/dbf-loader.spec.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import test from 'tape-promise/tape'; import {setLoaderOptions, fetchFile, parse} from '@loaders.gl/core'; import {DBFLoader} from '@loaders.gl/shapefile'; @@ -38,7 +41,7 @@ test('Shapefile JS DBF tests', async (t) => { const {features} = await response.json(); for (let i = 0; i < features.length; i++) { - t.deepEqual(output[i], features[i].properties, testFileName); + t.deepEqual(output.data[i], features[i].properties, testFileName); } } diff --git a/modules/shapefile/test/shapefile-loader.spec.js b/modules/shapefile/test/shapefile-loader.spec.ts similarity index 93% rename from modules/shapefile/test/shapefile-loader.spec.js rename to modules/shapefile/test/shapefile-loader.spec.ts index 0cc2cb0566..77e432e70d 100644 --- a/modules/shapefile/test/shapefile-loader.spec.js +++ b/modules/shapefile/test/shapefile-loader.spec.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import test from 'tape-promise/tape'; import { setLoaderOptions, @@ -52,11 +55,10 @@ test('ShapefileLoader#load (from browser File objects)', async (t) => { for (const testFileName in SHAPEFILE_JS_TEST_FILES) { const fileList = SHAPEFILE_JS_TEST_FILES[testFileName]; const fileSystem = new BrowserFileSystem(fileList); - const {fetch} = fileSystem; const filename = `${testFileName}.shp`; - // @ts-ignore - const data = await load(filename, ShapefileLoader, {fetch}); - // t.comment(`${filename}: ${JSON.stringify(data).slice(0, 70)}`); + // eslint-disable-next-line @typescript-eslint/unbound-method + const data = await load(filename, ShapefileLoader, {fetch: fileSystem.fetch}); + t.comment(`${filename}: ${JSON.stringify(data).slice(0, 70)}`); testShapefileData(t, testFileName, data); } @@ -98,6 +100,7 @@ test('ShapefileLoader#load and reproject (from files or URLs)', async (t) => { const shpFeature = data.data[i]; const jsonFeature = json.features[i]; const jsonPointGeom = projection.project(jsonFeature.geometry.coordinates); + // @ts-ignore coordinates does not exist on GeometryCollection, but on all others tapeEqualsEpsilon(t, shpFeature.geometry.coordinates, jsonPointGeom, 0.00001); } @@ -155,7 +158,7 @@ test('ShapefileLoader#loadInBatches(File)', async (t) => { const batches = await loadInBatches(file, ShapefileLoader, {fetch: fileSystem.fetch}); let data; for await (const batch of batches) { - data = batch; + data = batch.data; } await testShapefileData(t, testFileName, data); } @@ -169,7 +172,7 @@ test('ShapefileLoader#loadInBatches when options.metadata: true', async (t) => { const batches = await loadInBatches(filename, ShapefileLoader, {metadata: true}); let data; for await (const batch of batches) { - data = batch; + data = batch.data; // t.comment(`${filename}: ${JSON.stringify(data).slice(0, 70)}`); } await testShapefileData(t, testFileName, data); @@ -179,7 +182,7 @@ test('ShapefileLoader#loadInBatches when options.metadata: true', async (t) => { async function getFileList(testFileName) { const EXTENSIONS = ['.shp', '.shx', '.dbf', '.cpg', '.prj']; - const fileList = []; + const fileList: File[] = []; for (const extension of EXTENSIONS) { const filename = `${testFileName}${extension}`; const response = await fetchFile(`${SHAPEFILE_JS_DATA_FOLDER}/${filename}`); diff --git a/modules/shapefile/test/shapefile.bench.js b/modules/shapefile/test/shapefile.bench.ts similarity index 86% rename from modules/shapefile/test/shapefile.bench.js rename to modules/shapefile/test/shapefile.bench.ts index 8981ef102e..ed3c7ea0ec 100644 --- a/modules/shapefile/test/shapefile.bench.js +++ b/modules/shapefile/test/shapefile.bench.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import {fetchFile, load} from '@loaders.gl/core'; import {ShapefileLoader} from '@loaders.gl/shapefile'; @@ -12,7 +15,7 @@ export default async function shapefileLoaderBench(suite) { suite.group('ShapefileLoader'); suite.addAsync( - `parse(ShapefileLoader without worker)`, + 'parse(ShapefileLoader without worker)', {multiplier: 77, unit: 'MB'}, async () => { await load(arrayBuffer.slice(0), ShapefileLoader, {worker: false}); diff --git a/modules/shapefile/test/shp-loader.spec.js b/modules/shapefile/test/shp-loader.spec.ts similarity index 95% rename from modules/shapefile/test/shp-loader.spec.js rename to modules/shapefile/test/shp-loader.spec.ts index ca5ea2e89e..8bab3dd345 100644 --- a/modules/shapefile/test/shp-loader.spec.js +++ b/modules/shapefile/test/shp-loader.spec.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import test from 'tape-promise/tape'; import {setLoaderOptions, load, fetchFile} from '@loaders.gl/core'; import {geojsonToBinary} from '@loaders.gl/gis'; @@ -86,13 +89,13 @@ test('SHPLoader#_maxDimensions', async (t) => { const output2d = await load(`${SHAPEFILE_JS_DATA_FOLDER}/${POINT_Z_TEST_FILE}.shp`, SHPLoader, { shp: {_maxDimensions: 2} }); - t.equal(output2d.geometries[0].positions.size, 2); + t.equal(output2d.geometries[0]?.positions.size, 2); const defaultOutput = await load( `${SHAPEFILE_JS_DATA_FOLDER}/${POINT_Z_TEST_FILE}.shp`, SHPLoader ); - t.equal(defaultOutput.geometries[0].positions.size, 4); + t.equal(defaultOutput.geometries[0]?.positions.size, 4); t.end(); }); diff --git a/modules/shapefile/test/shp.bench.js b/modules/shapefile/test/shp.bench.ts similarity index 87% rename from modules/shapefile/test/shp.bench.js rename to modules/shapefile/test/shp.bench.ts index 9aff4138db..02a4e1f060 100644 --- a/modules/shapefile/test/shp.bench.js +++ b/modules/shapefile/test/shp.bench.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import {SHPLoader} from '@loaders.gl/shapefile'; import {parse, parseInBatches, fetchFile} from '@loaders.gl/core'; @@ -12,12 +15,12 @@ export default async function shpLoaderBench(suite) { suite.group('SHPLoader'); suite.addAsync( - `parse(SHPLoader without worker)`, + 'parse(SHPLoader without worker)', {multiplier: 77, unit: 'MB'}, async () => await parse(arrayBuffer, SHPLoader, {worker: false}) ); suite.addAsync( - `parseInBatches(SHPLoader without worker)`, + 'parseInBatches(SHPLoader without worker)', {multiplier: 77, unit: 'MB'}, async () => await parseInBatches(arrayBuffer, SHPLoader, {worker: false}) ); diff --git a/modules/shapefile/test/streaming/binary-chunk-reader.spec.ts b/modules/shapefile/test/streaming/binary-chunk-reader.spec.ts index 29114a6a7e..5148f7d9fd 100644 --- a/modules/shapefile/test/streaming/binary-chunk-reader.spec.ts +++ b/modules/shapefile/test/streaming/binary-chunk-reader.spec.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import test from 'tape-promise/tape'; import {_BinaryChunkReader as BinaryChunkReader} from '@loaders.gl/shapefile'; diff --git a/modules/shapefile/test/streaming/zip-batch-iterators.spec.ts b/modules/shapefile/test/streaming/zip-batch-iterators.spec.ts index 9834448329..6aa0082115 100644 --- a/modules/shapefile/test/streaming/zip-batch-iterators.spec.ts +++ b/modules/shapefile/test/streaming/zip-batch-iterators.spec.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import test from 'tape-promise/tape'; import type {ObjectRowTableBatch, ArrayRowTableBatch} from '@loaders.gl/schema'; import {_zipBatchIterators as zipBatchIterators} from '@loaders.gl/shapefile'; diff --git a/modules/shapefile/wip/parse-dbf-atomic.ts b/modules/shapefile/wip/parse-dbf-atomic.ts index 0fe693d3f5..796626b746 100644 --- a/modules/shapefile/wip/parse-dbf-atomic.ts +++ b/modules/shapefile/wip/parse-dbf-atomic.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import BinaryReader from '../../streaming/binary-reader'; const LITTLE_ENDIAN = true; diff --git a/modules/shapefile/wip/parse-shp-atomic.ts b/modules/shapefile/wip/parse-shp-atomic.ts index 52df7dd4d0..e3059388eb 100644 --- a/modules/shapefile/wip/parse-shp-atomic.ts +++ b/modules/shapefile/wip/parse-shp-atomic.ts @@ -1,3 +1,6 @@ +// loaders.gl, MIT license +// Copyright (c) vis.gl contributors + import BinaryReader from '../../streaming/binary-reader'; import {parseRecord} from '../parse-shp-geometry';