Skip to content

Commit

Permalink
[Bug] fix file handler row parsing to support single geojson feature (k…
Browse files Browse the repository at this point in the history 
…eplergl#1212)

Signed-off-by: Shan He <[email protected]>
  • Loading branch information
@heshan0131
heshan0131 authored Jul 31, 2020
1 parent 955c611 commit ff01c4a
Show file tree
Hide file tree
Showing 3 changed files with 151 additions and 3 deletions.
6 changes: 4 additions & 2 deletions src/processors/file-handler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ export async function* readBatch(asyncIterator, fileName) {
for await (const batch of asyncIterator) {
// Last batch will have this special type and will provide all the root
// properties of the parsed document.
// Only json parse will have `FINAL_RESULT`
if (batch.batchType === BATCH_TYPE.FINAL_RESULT) {
if (batch.container) {
result = {...batch.container};
Expand All @@ -112,8 +113,9 @@ export async function* readBatch(asyncIterator, fileName) {
if (batch.jsonpath && batch.jsonpath.length > 1) {
const streamingPath = new _JSONPath(batch.jsonpath);
streamingPath.setFieldAtPath(result, batches);
} else {
// The streamed object is a ROW JSON-batch
} else if (batch.jsonpath && batch.jsonpath.length === 1) {
// The streamed object is a ROW JSON-batch (jsonpath = '$')
// row objects
result = batches;
}
} else {
Expand Down
100 changes: 99 additions & 1 deletion test/browser/file-handler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ import {processFileData, readFileInBatches} from 'processors/file-handler';
import {csvWithNull} from '../node/processors/file-handler-fixtures';
import {dataWithNulls, testFields, parsedDataWithNulls} from 'test/fixtures/test-csv-data';
import geojsonString, {
featureString,
processedFeature,
processedFeatureRows,
processedFeatureFields,
processedFields as geojsonFields,
processedRows as geojsonRows
} from 'test/fixtures/geojson-style';
Expand Down Expand Up @@ -144,7 +148,7 @@ test('#file-handler -> readFileInBatches.csv -> processFileData', async t => {
t.end();
});

test('#file-handler -> readFileInBatches.geoJson -> processFileData', async t => {
test('#file-handler -> readFileInBatches.GeoJSON FeatureCollection -> processFileData', async t => {
const geojsonFile = new File([geojsonString], 'text-data-1.geojson', {type: ''});
const gen = await readFileInBatches({file: geojsonFile, fileCache: []});

Expand Down Expand Up @@ -266,6 +270,100 @@ test('#file-handler -> readFileInBatches.geoJson -> processFileData', async t =>
t.end();
});

test('#file-handler -> readFileInBatches.GeoJSON Single Feature -> processFileData', async t => {
const geojsonFile = new File([featureString], 'text-data-1.geojson', {type: ''});
const gen = await readFileInBatches({file: geojsonFile, fileCache: []});

// metadata batch
const batch1 = await gen.next();

const expected1 = {
value: {
batchType: 'metadata',
metadata: {_loader: {}, _context: {}},
data: [],
bytesUsed: 0,
progress: {rowCount: 0, rowCountInBatch: 0, percent: 0},
fileName: 'text-data-1.geojson'
},
done: false
};

t.deepEqual(
Object.keys(batch1.value).sort(),
Object.keys(expected1.value).sort(),
'value should have same keyss'
);

t.equal(batch1.value.batchType, expected1.value.batchType, 'batch1.batchType should be the same');
t.equal(batch1.value.fileName, expected1.value.fileName, 'batch1.fileName should be the same');
t.deepEqual(batch1.value.data, expected1.value.data, 'batch1.data should be the same');
t.deepEqual(
batch1.value.progress,
expected1.value.progress,
'batch1.progress should be the same'
);

// final result batch
const batch2 = await gen.next();
const expected2 = {
value: {
batchType: 'final-result',
container: processedFeature,
jsonpath: null,
data: processedFeature,
schema: null,
progress: {rowCount: 0, rowCountInBatch: 0},
fileName: 'text-data-1.geojson'
},
done: false
};

t.deepEqual(
batch2.value.batchType,
expected2.value.batchType,
'batch2 batchType should be a final-result'
);
t.deepEqual(
batch2.value.data,
expected2.value.data,
'batch2 data should be a single geojson feature'
);
t.deepEqual(
batch2.value.container,
expected2.value.container,
'batch2 container should be a single geojson feature'
);
t.equal(batch2.value.jsonpath, expected2.value.jsonpath, 'batch2 jsonpath should be null');
t.deepEqual(batch2.value.progress, expected2.value.progress, 'batch2 progress should be correct');

const batch3 = await gen.next();
t.deepEqual(batch3, {value: undefined, done: true}, 'batch3 should be done');

// process geojson data received
const processed = await processFileData({content: batch2.value, fileCache: []});
const expectedInfo = {label: 'text-data-1.geojson', format: 'geojson'};

t.equal(processed.length, 1, 'processFileData should return 1 result');
t.ok(processed[0].info, 'processFileData should have info');
t.ok(processed[0].data, 'processFileData should have data');
t.deepEqual(processed[0].info, expectedInfo, 'info should be correct');
const {fields, rows} = processed[0].data;

fields.forEach((f, i) => {
t.deepEqual(
f,
processedFeatureFields[i],
`should process correct geojson field ${processedFeatureFields[i].name}`
);
});
rows.forEach((r, i) => {
t.deepEqual(r, processedFeatureRows[i], `should process geojson row ${i} correctly`);
});

t.end();
});

test('#file-handler -> readFileInBatches.row -> processFileData', async t => {
const fileName = 'row-data.json';
const rowFile = new File([rowDataString], fileName, {type: ''});
Expand Down
48 changes: 48 additions & 0 deletions test/fixtures/geojson-style.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,54 @@
const geojsonString = `{"type":"FeatureCollection","features":[{"geometry":{"type":"Point","coordinates":[-105.15471672508885,39.98626910199207,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"radius":1,"id":"a1398a11-d1ce-421c-bf66-a456ff525de9"}},{"geometry":{"type":"Point","coordinates":[-105.1549804351595,39.98397605319894,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"94ed2c0f-27ce-416e-b3b4-6c00954b41f0"}},{"geometry":{"type":"Point","coordinates":[-105.15478982047146,39.98296589543148,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"radius":3,"id":"1f59be4c-82e8-4644-b3cf-4c1c0510cbb2"}},{"geometry":{"type":"Point","coordinates":[-105.15449343321012,39.98437157892626,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"radius":5,"id":"de9210a0-c48c-41bf-8463-4b0734d402c0"}},{"geometry":{"type":"Point","coordinates":[-105.15484576666667,39.98312416666666,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"9438eea5-dde0-4e5e-bdf5-7420eedbc419"}},{"geometry":{"type":"Point","coordinates":[-105.15494596666667,39.98422703333333,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"radius":0.5,"id":"9ecefa7c-4fc6-46b7-936c-580a6084ff47"}},{"geometry":{"type":"Point","coordinates":[-105.15522212737501,39.98433057912913,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"696a33f6-3ff2-45a9-8a11-ab541aa2152f"}},{"geometry":{"type":"Point","coordinates":[-105.15447046666667,39.9834028,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"a5b27f57-37a4-4576-ac4e-24dfb57730c3"}},{"geometry":{"type":"Point","coordinates":[-105.15487273333333,39.98379046666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"7a640dbe-e0ca-4c50-a43a-b65ec33305b5"}},{"geometry":{"type":"Point","coordinates":[-105.154786,39.986418799999996,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"154cc349-c8b3-4b41-a008-364df1e6d83d"}},{"geometry":{"type":"Point","coordinates":[-105.15456956666667,39.984760566666665,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"932d45a2-bc4b-4825-9b9f-70b2415753b5"}},{"geometry":{"type":"Point","coordinates":[-105.15503543214001,39.983469561355626,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"a178d24c-1ea3-46cd-8762-3669d7b9d722"}},{"geometry":{"type":"Point","coordinates":[-105.15462816666667,39.984541766666666,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"41f8b94e-266c-450a-a7dc-60c159370ddb"}},{"geometry":{"type":"Point","coordinates":[-105.15494077274344,39.98493993797259,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"99ee26ca-13e1-4bef-bcd2-12cfc290d6ae"}},{"geometry":{"type":"Point","coordinates":[-105.15451193333332,39.98379226666666,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"de5103bb-2808-46b4-8012-76ceb5511fa9"}},{"geometry":{"type":"Point","coordinates":[-105.1547839,39.98589206666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"c5caef3d-abc9-4044-bb14-2f5999979d8d"}},{"geometry":{"type":"Point","coordinates":[-105.15513263333332,39.98269536666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"e0162a54-e2b2-4076-be24-4f550f0ac651"}},{"geometry":{"type":"Point","coordinates":[-105.1546226,39.98516726666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"e53ab272-7f6a-49b8-93f1-693bb57aa9f8"}},{"geometry":{"type":"Point","coordinates":[-105.1545185,39.98417893333333,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"fdc3fff3-5c35-430b-8bb4-9517473a8dc1"}},{"geometry":{"type":"Point","coordinates":[-105.15476356666666,39.98555806666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"f342e7bd-771d-4bf1-ad57-40d435f54fa0"}},{"geometry":{"type":"Point","coordinates":[-105.15468952992738,39.985742351897976,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"dbda33a4-b194-40e8-88a6-5e8816a97377"}},{"geometry":{"type":"Point","coordinates":[-105.1545091,39.983995666666665,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"5564b2fb-8368-4f81-a8b4-bebb30cbaaed"}},{"geometry":{"type":"Point","coordinates":[-105.15508416666667,39.985163,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"835f956f-6a86-4a7c-b1d6-3f32e4955a31"}},{"geometry":{"type":"Point","coordinates":[-105.15513636666667,39.983126066666664,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"8f8effff-f89d-417c-8e44-c8561f9f1eec"}},{"geometry":{"type":"Point","coordinates":[-105.15478573333333,39.98606776666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"59552665-bdd6-4c0c-bd0d-5ee79ea4848b"}},{"geometry":{"type":"Point","coordinates":[-105.1547872,39.9866052,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"32df5401-38d1-4e96-8d1e-a29d2c7c3955"}}]}`;
export default geojsonString;

export const featureString = `{"type":"Feature","geometry":{"type":"LineString","coordinates":[[41.8817441,-87.6335398],[41.88237441,-87.6331298]]},"properties":{"lat":41.8817441,"lng":-87.6335398,"icon":"place"}}`;

export const processedFeature = {
type: 'Feature',
geometry: {
type: 'LineString',
coordinates: [
[41.8817441, -87.6335398],
[41.88237441, -87.6331298]
]
},
properties: {
lat: 41.8817441,
lng: -87.6335398,
icon: 'place'
}
};

export const processedFeatureFields = [
{name: '_geojson', format: '', tableFieldIndex: 1, type: 'geojson', analyzerType: 'GEOMETRY'},
{name: 'lat', format: '', tableFieldIndex: 2, type: 'real', analyzerType: 'FLOAT'},
{name: 'lng', format: '', tableFieldIndex: 3, type: 'real', analyzerType: 'FLOAT'},
{name: 'icon', format: '', tableFieldIndex: 4, type: 'string', analyzerType: 'STRING'}
];

export const processedFeatureRows = [
[
{
type: 'Feature',
geometry: {
type: 'LineString',
coordinates: [
[41.8817441, -87.6335398],
[41.88237441, -87.6331298]
]
},
properties: {
lat: 41.8817441,
lng: -87.6335398,
icon: 'place'
}
},
41.8817441,
-87.6335398,
'place'
]
];

export const processedFields = [
{name: '_geojson', format: '', tableFieldIndex: 1, type: 'geojson', analyzerType: 'GEOMETRY'},
{name: 'fillColor', format: '', tableFieldIndex: 2, type: 'geojson', analyzerType: 'ARRAY'},
Expand Down

0 comments on commit ff01c4a

Please sign in to comment.