From c51e64a28414fe490c637ebd66c5760b56033f37 Mon Sep 17 00:00:00 2001 From: Aymeric DUTREMBLE Date: Wed, 20 Sep 2023 12:20:28 +0200 Subject: [PATCH] feature(VectorTileSource): add support for multiple source --- src/Source/Source.js | 6 +- src/Source/VectorTilesSource.js | 86 ++++- test/functional/vector_tile_3d_mesh_mapbox.js | 6 +- test/unit/vectortiles.js | 356 +++++++++++------- 4 files changed, 307 insertions(+), 147 deletions(-) diff --git a/src/Source/Source.js b/src/Source/Source.js index cb6b0f33aa..c8183e0ca6 100644 --- a/src/Source/Source.js +++ b/src/Source/Source.js @@ -68,7 +68,6 @@ class /* istanbul ignore next */ ParsingOptions {} function fetchSourceData(source, extent) { const url = source.urlFromExtent(extent); - return source.fetcher(url, source.networkOptions).then((f) => { f.extent = extent; return f; @@ -190,8 +189,9 @@ class Source extends InformationsData { let features = cache.getByArray(key); if (!features) { // otherwise fetch/parse the data - features = cache.setByArray(fetchSourceData(this, extent).then(file => this.parser(file, { out, in: this }), - err => this.handlingError(err)), key); + features = cache.setByArray(fetchSourceData(this, extent) + .then(file => this.parser(file, { out, in: this }), + err => this.handlingError(err)), key); /* istanbul ignore next */ if (this.onParsedFile) { features.then((feat) => { diff --git a/src/Source/VectorTilesSource.js b/src/Source/VectorTilesSource.js index 9d2f69192d..77c1a0531b 100644 --- a/src/Source/VectorTilesSource.js +++ b/src/Source/VectorTilesSource.js @@ -1,6 +1,7 @@ import { featureFilter } from '@mapbox/mapbox-gl-style-spec'; import Style from 'Core/Style'; import TMSSource from 'Source/TMSSource'; +import URLBuilder from 'Provider/URLBuilder'; import Fetcher from 'Provider/Fetcher'; import urlParser from 'Parser/MapBoxUrlParser'; @@ -8,6 +9,22 @@ function toTMSUrl(url) { return url.replace(/\{/g, '${'); } +function fetchSourceData(source, url) { + return source.fetcher(url, source.networkOptions) + .then(f => f, err => source.handlingError(err)); +} + +function mergeCollections(collections) { + const collection = collections[0]; + collections.forEach((col, index) => { + if (index === 0) { return; } + col.features.forEach((feature) => { + collection.features.push(feature); + }); + }); + return collection; +} + /** * @classdesc * VectorTilesSource are object containing informations on how to fetch vector @@ -86,9 +103,6 @@ class VectorTilesSource extends TMSSource { return style; }).then((style) => { - const s = Object.keys(style.sources)[0]; - const os = style.sources[s]; - style.layers.forEach((layer, order) => { layer.sourceUid = this.uid; if (layer.type === 'background') { @@ -113,17 +127,34 @@ class VectorTilesSource extends TMSSource { }); if (this.url == '.') { - if (os.url) { - const urlSource = urlParser.normalizeSourceURL(os.url, this.accessToken); - return Fetcher.json(urlSource, this.networkOptions).then((tileJSON) => { - if (tileJSON.tiles[0]) { - this.url = toTMSUrl(tileJSON.tiles[0]); - } - }); - } else if (os.tiles[0]) { - this.url = toTMSUrl(os.tiles[0]); - } + const TMSUrlList = Object.values(style.sources).map((source) => { + if (source.url) { + const urlSource = urlParser.normalizeSourceURL(source.url, this.accessToken); + return Fetcher.json(urlSource, this.networkOptions).then((tileJSON) => { + if (tileJSON.tiles[0]) { + return toTMSUrl(tileJSON.tiles[0]); + } + }); + } else if (source.tiles) { + return Promise.resolve(toTMSUrl(source.tiles[0])); + } + return Promise.reject(); + }); + return Promise.all(TMSUrlList); } + return (Promise.resolve([this.url])); + }).then((TMSUrlList) => { + this.url = new Set(TMSUrlList); + }); + } + + urlFromExtent(extent) { + return this.url.map((url) => { + const options = { + tileMatrixCallback: this.tileMatrixCallback, + url, + }; + return URLBuilder.xyz(extent, options); }); } @@ -136,6 +167,35 @@ class VectorTilesSource extends TMSSource { } } } + + loadData(extent, out) { + const cache = this._featuresCaches[out.crs]; + const key = this.requestToKey(extent); + // try to get parsed data from cache + let features = cache.getByArray(key); + if (!features) { + // otherwise fetch/parse the data + features = cache.setByArray( + Promise.all(this.urlFromExtent(extent).map(url => + fetchSourceData(this, url) + .then((file) => { + file.extent = extent; + return this.parser(file, { out, in: this }); + }), + )).then(collections => mergeCollections(collections), + err => this.handlingError(err)), key); + + /* istanbul ignore next */ + if (this.onParsedFile) { + features.then((feat) => { + this.onParsedFile(feat); + console.warn('Source.onParsedFile was deprecated'); + return feat; + }); + } + } + return features; + } } export default VectorTilesSource; diff --git a/test/functional/vector_tile_3d_mesh_mapbox.js b/test/functional/vector_tile_3d_mesh_mapbox.js index 786bb98edb..903514c45b 100644 --- a/test/functional/vector_tile_3d_mesh_mapbox.js +++ b/test/functional/vector_tile_3d_mesh_mapbox.js @@ -1,6 +1,6 @@ const assert = require('assert'); -describe('vector_tile_3d_mesh_mapbox', function _() { +describe('vector_tile_3d_mesh_mapbox', function _describe() { let result; before(async () => { result = await loadExample('examples/vector_tile_3d_mesh_mapbox.html', this.fullTitle()); @@ -10,8 +10,8 @@ describe('vector_tile_3d_mesh_mapbox', function _() { assert.ok(result); }); - it('should correctly load building features on a given TMS tile', async function _2() { - const featuresCollection = await page.evaluate(async function _3() { + it('should correctly load building features on a given TMS tile', async function _it() { + const featuresCollection = await page.evaluate(async function _() { const layers = view.getLayers(l => l.source && l.source.isVectorSource); const res = await layers[0].source.loadData({ zoom: 15, row: 11634, col: 16859 }, { crs: 'EPSG:4978', source: { crs: 'TMS:3857' } }); return res; diff --git a/test/unit/vectortiles.js b/test/unit/vectortiles.js index 4464ab319d..c473a10e81 100644 --- a/test/unit/vectortiles.js +++ b/test/unit/vectortiles.js @@ -10,11 +10,13 @@ import sinon from 'sinon'; import style from '../data/vectortiles/style.json'; import tilejson from '../data/vectortiles/tilejson.json'; import sprite from '../data/vectortiles/sprite.json'; +import mapboxStyle from '../data/mapboxMulti.json'; const resources = { 'test/data/vectortiles/style.json': style, 'https://test/tilejson.json': tilejson, 'https://test/sprite.json': sprite, + 'https://api.mapbox.com/v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v7.json': mapboxStyle, }; function parse(pbf, layers) { @@ -27,7 +29,7 @@ function parse(pbf, layers) { crs: 'EPSG:3857', }, }); -} +}; describe('Vector tiles', function () { let stub; @@ -89,159 +91,257 @@ describe('Vector tiles', function () { done(); }).catch(done); }); +}); - describe('VectorTilesSource', function () { - it('throws an error because no style was provided', () => { - assert.throws(() => new VectorTilesSource({}), { - name: 'Error', - message: 'New VectorTilesSource: style is required', +describe('VectorTilesSource', function () { + let stub; + before(function () { + stub = sinon.stub(Fetcher, 'json') + .callsFake((url) => { + url = url.split('?')[0]; + return Promise.resolve(JSON.parse(resources[url])); }); + }); + after(function () { + stub.restore(); + }); + + it('throws an error because no style was provided', () => { + assert.throws(() => new VectorTilesSource({}), { + name: 'Error', + message: 'New VectorTilesSource: style is required', }); + }); - it('reads tiles URL from the style', (done) => { - const source = new VectorTilesSource({ - style: { - sources: { tilesurl: { tiles: ['http://server.geo/{z}/{x}/{y}.pbf'] } }, - layers: [], - }, - }); - source.whenReady.then(() => { - // eslint-disable-next-line no-template-curly-in-string - assert.equal(source.url, 'http://server.geo/${z}/${x}/${y}.pbf'); - done(); - }).catch(done); + it('reads tiles URL directly from the style', (done) => { + const source = new VectorTilesSource({ + style: { + sources: { sourceTiles: { tiles: ['http://server.geo/{z}/{x}/{y}.pbf'] } }, + layers: [], + }, }); + source.whenReady.then(() => { + assert.equal(source.url.size, 1); + // eslint-disable-next-line no-template-curly-in-string + assert.ok(source.url.has('http://server.geo/${z}/${x}/${y}.pbf')); + done(); + }) + .catch(done); + }); - it('reads the background layer', (done) => { - const source = new VectorTilesSource({ - url: 'fakeurl', - style: { - sources: { tilejson: {} }, - layers: [{ type: 'background' }], - }, - }); - source.whenReady.then(() => { - assert.ok(source.backgroundLayer); + it('reads tiles URL from an url', (done) => { + const source = new VectorTilesSource({ + style: { + sources: { sourceUrl: { url: 'mapbox://mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v7' } }, + layers: [{ + id: 'building', + source: 'sourceUrl', + 'source-layer': 'building', + type: 'fill', + paint: { + 'fill-color': 'red', + }, + }], + }, + accessToken: 'pk.eyJ1IjoiZnRvcm9tYW5vZmYiLCJhIjoiY2xramhzM2xrMDVibjNpcGNvdGRlZWQ5YyJ9.NibhjJNVTxArsNSH4v_kIA', + }); + source.whenReady + .then(() => { + assert.equal(source.url, '.'); + assert.equal(source.urls.size, 1); done(); - }).catch(done); + }) + .catch(done); + }); + + it('reads the background layer', (done) => { + const source = new VectorTilesSource({ + url: 'fakeurl', + style: { + sources: { tilejson: {} }, + layers: [{ type: 'background' }], + }, }); + source.whenReady.then(() => { + assert.ok(source.backgroundLayer); + done(); + }) + .catch(done); + }); - it('creates styles and assigns filters', (done) => { - const source = new VectorTilesSource({ - url: 'fakeurl', - style: { - sources: { tilejson: {} }, - layers: [{ - id: 'land', - type: 'fill', - paint: { - 'fill-color': 'rgb(255, 0, 0)', - }, - }], - }, - }); - source.whenReady.then(() => { - assert.ok(source.styles.land); + it('creates styles and assigns filters', (done) => { + const source = new VectorTilesSource({ + url: 'fakeurl', + style: { + sources: { tilejson: {} }, + layers: [{ + id: 'land', + type: 'fill', + paint: { + 'fill-color': 'rgb(255, 0, 0)', + }, + }], + }, + }); + source.whenReady.then(() => { + assert.ok(source.styles.land); + assert.equal(source.styles.land.fill.color, 'rgb(255,0,0)'); + done(); + }) + .catch(done); + }); + + it('loads the style from a file', function _it(done) { + const source = new VectorTilesSource({ + style: 'test/data/vectortiles/style.json', + networkOptions: process.env.HTTPS_PROXY ? { agent: new HttpsProxyAgent(process.env.HTTPS_PROXY) } : {}, + }); + source.whenReady + .then(() => { assert.equal(source.styles.land.fill.color, 'rgb(255,0,0)'); + assert.equal(source.styles.land.fill.opacity, 1); + assert.equal(source.styles.land.zoom.min, 5); + assert.equal(source.styles.land.zoom.max, 13); done(); }).catch(done); + }); + + it('sets the correct Style#zoom.min', (done) => { + const source = new VectorTilesSource({ + url: 'fakeurl', + style: { + sources: { tilejson: {} }, + layers: [{ + // minzoom is 0 (default value) + id: 'first', + type: 'fill', + paint: { + 'fill-color': 'rgb(255, 0, 0)', + }, + }, { + // minzoom is 5 (specified) + id: 'second', + type: 'fill', + paint: { + 'fill-color': 'rgb(255, 0, 0)', + }, + minzoom: 5, + }, { + // minzoom is 4 (first stop) + // If a style have `stops` expression, should it be used to determine the min zoom? + id: 'third', + type: 'fill', + paint: { + 'fill-color': 'rgb(255, 0, 0)', + 'fill-opacity': { stops: [[4, 1], [7, 0.5]] }, + }, + }, { + // minzoom is 1 (first stop and no specified minzoom) + id: 'fourth', + type: 'fill', + paint: { + 'fill-color': 'rgb(255, 0, 0)', + 'fill-opacity': { stops: [[1, 1], [7, 0.5]] }, + }, + }, { + // minzoom is 4 (first stop is higher than specified) + id: 'fifth', + type: 'fill', + paint: { + 'fill-color': 'rgb(255, 0, 0)', + 'fill-opacity': { stops: [[4, 1], [7, 0.5]] }, + }, + minzoom: 3, + }], + }, }); - it('loads the style from a file', function _it(done) { + source.whenReady.then(() => { + assert.equal(source.styles.first.zoom.min, 0); + assert.equal(source.styles.second.zoom.min, 5); + assert.equal(source.styles.third.zoom.min, 0); + assert.equal(source.styles.fourth.zoom.min, 0); + assert.equal(source.styles.fifth.zoom.min, 3); + done(); + }) + .catch(done); + }); + + it('Vector tile source mapbox url', () => { + const accessToken = 'pk.xxxxx'; + const baseurl = 'mapbox://styles/mapbox/outdoors-v11'; + + const styleUrl = urlParser.normalizeStyleURL(baseurl, accessToken); + assert.ok(styleUrl.startsWith('https://api.mapbox.com')); + assert.ok(styleUrl.endsWith(accessToken)); + + const spriteUrl = urlParser.normalizeSpriteURL(baseurl, '', '.json', accessToken); + assert.ok(spriteUrl.startsWith('https')); + assert.ok(spriteUrl.endsWith(accessToken)); + assert.ok(spriteUrl.includes('sprite.json')); + + const imgUrl = urlParser.normalizeSpriteURL(baseurl, '', '.png', accessToken); + assert.ok(imgUrl.includes('sprite.png')); + + const url = 'mapbox://mapbox.mapbox-streets-v8,mapbox.mapbox-terrain-v2'; + const urlSource = urlParser.normalizeSourceURL(url, accessToken); + assert.ok(urlSource.startsWith('https')); + assert.ok(urlSource.endsWith(accessToken)); + assert.ok(urlSource.includes('.json')); + }); + + describe('multisource', function () { + it('2 sources with different url tiles', (done) => { const source = new VectorTilesSource({ - style: 'test/data/vectortiles/style.json', + style: { + sources: { + source1: { + type: 'vector', + tiles: ['http://server.geo/{z}/{x}/{y}.pbf'], + }, + source2: { + type: 'vector', + tiles: ['http://server.geo2/{z}/{x}/{y}.pbf'], + }, + }, + layers: [], + }, }); source.whenReady .then(() => { - assert.equal(source.styles.land.fill.color, 'rgb(255,0,0)'); - assert.equal(source.styles.land.fill.opacity, 1); - assert.equal(source.styles.land.zoom.min, 5); - assert.equal(source.styles.land.zoom.max, 13); + assert.equal(source.url.size, 2); + // eslint-disable-next-line no-template-curly-in-string + assert.ok(source.url.has('http://server.geo/${z}/${x}/${y}.pbf')); + // eslint-disable-next-line no-template-curly-in-string + assert.ok(source.url.has('http://server.geo2/${z}/${x}/${y}.pbf')); done(); - }).catch(done); + }) + .catch(done); }); - - it('sets the correct Style#zoom.min', (done) => { + it('2 sources with same url tiles', (done) => { const source = new VectorTilesSource({ - url: 'fakeurl', style: { - sources: { tilejson: {} }, - layers: [{ - // minzoom is 0 (default value) - id: 'first', - type: 'fill', - paint: { - 'fill-color': 'rgb(255, 0, 0)', - }, - }, { - // minzoom is 5 (specified) - id: 'second', - type: 'fill', - paint: { - 'fill-color': 'rgb(255, 0, 0)', - }, - minzoom: 5, - }, { - // minzoom is 4 (first stop) - // If a style have `stops` expression, should it be used to determine the min zoom? - id: 'third', - type: 'fill', - paint: { - 'fill-color': 'rgb(255, 0, 0)', - 'fill-opacity': { stops: [[4, 1], [7, 0.5]] }, - }, - }, { - // minzoom is 1 (first stop and no specified minzoom) - id: 'fourth', - type: 'fill', - paint: { - 'fill-color': 'rgb(255, 0, 0)', - 'fill-opacity': { stops: [[1, 1], [7, 0.5]] }, + sources: { + source1: { + type: 'vector', + tiles: ['http://server.geo/{z}/{x}/{y}.pbf'], }, - }, { - // minzoom is 4 (first stop is higher than specified) - id: 'fifth', - type: 'fill', - paint: { - 'fill-color': 'rgb(255, 0, 0)', - 'fill-opacity': { stops: [[4, 1], [7, 0.5]] }, + source2: { + type: 'vector', + tiles: ['http://server.geo/{z}/{x}/{y}.pbf'], }, - minzoom: 3, - }], + }, + layers: [], }, }); - - source.whenReady.then(() => { - assert.equal(source.styles.first.zoom.min, 0); - assert.equal(source.styles.second.zoom.min, 5); - assert.equal(source.styles.third.zoom.min, 0); - assert.equal(source.styles.fourth.zoom.min, 0); - assert.equal(source.styles.fifth.zoom.min, 3); - done(); - }).catch(done); - }); - - it('Vector tile source mapbox url', () => { - const accessToken = 'pk.xxxxx'; - const baseurl = 'mapbox://styles/mapbox/outdoors-v11'; - - const styleUrl = urlParser.normalizeStyleURL(baseurl, accessToken); - assert.ok(styleUrl.startsWith('https://api.mapbox.com')); - assert.ok(styleUrl.endsWith(accessToken)); - - const spriteUrl = urlParser.normalizeSpriteURL(baseurl, '', '.json', accessToken); - assert.ok(spriteUrl.startsWith('https')); - assert.ok(spriteUrl.endsWith(accessToken)); - assert.ok(spriteUrl.includes('sprite.json')); - - const imgUrl = urlParser.normalizeSpriteURL(baseurl, '', '.png', accessToken); - assert.ok(imgUrl.includes('sprite.png')); - - const url = 'mapbox://mapbox.mapbox-streets-v8,mapbox.mapbox-terrain-v2'; - const urlSource = urlParser.normalizeSourceURL(url, accessToken); - assert.ok(urlSource.startsWith('https')); - assert.ok(urlSource.endsWith(accessToken)); - assert.ok(urlSource.includes('.json')); + source.whenReady + .then(() => { + assert.equal(source.url.size, 1); + // eslint-disable-next-line no-template-curly-in-string + assert.ok(source.url.has('http://server.geo/${z}/${x}/${y}.pbf')); + done(); + }) + .catch(done); }); }); });