From 6272c1783544f4c99fb0222e39bb138fcfb19bbb Mon Sep 17 00:00:00 2001 From: Pascal Barth Date: Thu, 14 Nov 2024 09:21:40 +0100 Subject: [PATCH] PB-1028 : fix COG loading after rebase error While working on the latest PR for PB-1028 I made some rebase mistake and didn't detect them before merging. COG files were loaded entirely once again... So here's the fix, with some additional benefits. I tried to simplify the FileAPI functions, the ones that load files (either through a simple GET or through service-proxy) I wasn't possible, with the addition of COG to the mix, to keep FileParser to decide to load content on its own, so I created a function that get the MIME type (and other relevant info on the file) so that each time we need to load a file, we can decide which way (of loading) we want to go (proxy or not proxy) --- src/api/file-proxy.api.js | 2 +- src/api/files.api.js | 128 +++++--------- .../CloudOptimizedGeoTIFFParser.class.js | 23 +-- .../ImportFile/parser/FileParser.class.js | 123 ++++++++----- .../ImportFile/parser/GPXParser.class.js | 3 +- .../ImportFile/parser/KMLParser.class.js | 5 +- .../ImportFile/parser/KMZParser.class.js | 7 +- .../advancedTools/ImportFile/parser/index.js | 167 +++++++----------- src/store/plugins/load-cog-extent.plugin.js | 13 +- src/store/plugins/load-gpx-data.plugin.js | 13 +- src/store/plugins/load-kml-kmz-data.plugin.js | 28 ++- tests/cypress/tests-e2e/print.cy.js | 12 +- 12 files changed, 238 insertions(+), 286 deletions(-) diff --git a/src/api/file-proxy.api.js b/src/api/file-proxy.api.js index d1ef9f0bb..f30d540b2 100644 --- a/src/api/file-proxy.api.js +++ b/src/api/file-proxy.api.js @@ -74,7 +74,7 @@ export function unProxifyUrl(proxifiedUrl) { * @param {String} fileUrl * @returns {Promise} */ -export async function getContentThroughServiceProxy(fileUrl) { +export async function getFileContentThroughServiceProxy(fileUrl) { const proxifyGetResponse = await axios.get(proxifyUrl(fileUrl), { responseType: 'arraybuffer', }) diff --git a/src/api/files.api.js b/src/api/files.api.js index 40130d130..703330c41 100644 --- a/src/api/files.api.js +++ b/src/api/files.api.js @@ -5,7 +5,6 @@ import pako from 'pako' import { proxifyUrl } from '@/api/file-proxy.api' import { getServiceKmlBaseUrl } from '@/config/baseUrl.config' import log from '@/utils/logging' -import { isInternalUrl } from '@/utils/utils' /** * KML links @@ -298,102 +297,61 @@ export function loadKmlMetadata(kmlLayer) { } /** - * Loads the XML data from the file of a given KML layer, using the KML file URL of the layer. + * Load content of a file from a given URL as ArrayBuffer. * - * @param {KMLLayer} kmlLayer + * @param {string} url URL to fetch * @returns {Promise} */ -export function loadKmlData(kmlLayer) { - return new Promise((resolve, reject) => { - if (!kmlLayer) { - reject(new Error('Missing KML layer, cannot load data')) - } - if (!kmlLayer.kmlFileUrl) { - reject( - new Error(`No file URL defined in this KML layer, cannot load data ${kmlLayer.id}`) - ) - } - // The file might be a KMZ file, which is a zip archive. Reading zip archive as text - // is asking for trouble therefore we use ArrayBuffer - getFileFromUrl(kmlLayer.kmlFileUrl, { responseType: 'arraybuffer' }) - .then((response) => { - if (response.status === 200 && response.data) { - resolve(response.data) - } else { - const msg = `Incorrect response while getting KML file data for layer ${kmlLayer.id}` - log.error(msg, response) - reject(new Error(msg)) - } - }) - .catch((error) => { - const msg = `Failed to load KML data: ${error}` - log.error(msg) - reject(new Error(msg)) - }) - }) +export async function getFileContentFromUrl(url) { + const response = await axios.get(url, { responseType: 'arraybuffer' }) + return response.data } /** - * Generic function to load a file from a given URL. - * - * When the URL is not an internal url and it doesn't support CORS or use HTTP it is sent over a - * proxy. - * - * @param {string} url URL to fetch - * @param {Object} [options] - * @param {Number} [options.timeout] How long should the call wait before timing out - * @param {string} [options.responseType] Type of data that the server will respond with. Options - * are 'arraybuffer', 'document', 'json', 'text', 'stream'. Default is `json` - * @returns {Promise>} + * @typedef OnlineFileCompliance + * @property {String | null} mimeType + * @property {Boolean} supportsCORS If `true` it means that a HEAD request could go through CORS + * checks. + * @property {Boolean} supportsHTTPS */ -export async function getFileFromUrl(url, options = {}) { - const { timeout = null, responseType = null } = options - if (/^https?:\/\/localhost/.test(url) || isInternalUrl(url)) { - // don't go through proxy if it is on localhost or the internal server - return await axios.get(url, { timeout, responseType }) - } else if (url.startsWith('http://')) { - // HTTP request goes through the proxy - return await axios.get(proxifyUrl(url), { timeout, responseType }) - } - - // For other urls we need to check if they support CORS - let supportCORS = false - try { - // unfortunately we cannot do a real preflight call using options because browser don't - // allow to set the Access-Control-* headers ! Also there is no way to find out if a request - // is failing due to network reason or due to CORS issue, - // see https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS/Errors - // Therefore here we try to get the resource using head instead - await axios.head(url, { timeout }) - supportCORS = true - } catch (error) { - log.error( - `URL ${url} failed with ${error}. It might be due to CORS issue, ` + - `therefore the request will be fallback to the service-proxy` - ) - } - - if (supportCORS) { - // Server support CORS - return await axios.get(url, { timeout, responseType }) - } - // server don't support CORS use proxy - return await axios.get(proxifyUrl(url), { timeout, responseType }) -} /** - * Get a file MIME type through a HEAD request (and reading the Content-Type header returned by this + * Get a file MIME type through a HEAD request, and reading the Content-Type header returned by this * request. Returns `null` if the HEAD request failed, or if no Content-Type header is set. * - * @param url - * @returns {Promise} + * Will attempt to get the HEAD request through service-proxy if the first HEAD request failed. + * + * Will return if the first HEAD request was successful through a boolean called `supportCORS`, if + * this is `true` it means that the first HEAD request could go through CORS checks. + * + * Also returns a flag telling if the file supports HTTPS or not. + * + * @param {String} url + * @returns {Promise} */ -export async function getFileMimeType(url) { +export async function checkOnlineFileCompliance(url) { + const supportsHTTPS = url.startsWith('https://') + if (supportsHTTPS) { + try { + const headResponse = await axios.head(url) + return { + mimeType: headResponse.headers.get('content-type'), + supportsCORS: true, + supportsHTTPS, + } + } catch (error) { + log.error(`HEAD request on URL ${url} failed with`, error) + } + } try { - const headResponse = await axios.head(url) - return headResponse.headers.get('content-type') - } catch (error) { - log.error(`HEAD request on URL ${url} failed with`, error) - return null + const proxyHeadResponse = await axios.head(proxifyUrl(url)) + return { + mimeType: proxyHeadResponse.headers.get('content-type'), + supportsCORS: false, + supportsHTTPS, + } + } catch (errorWithProxy) { + log.error('HEAD request through proxy failed for URL', url, errorWithProxy) + return { mimeType: null, supportsCORS: false, supportsHTTPS } } } diff --git a/src/modules/menu/components/advancedTools/ImportFile/parser/CloudOptimizedGeoTIFFParser.class.js b/src/modules/menu/components/advancedTools/ImportFile/parser/CloudOptimizedGeoTIFFParser.class.js index 1260ab34d..a531a7b50 100644 --- a/src/modules/menu/components/advancedTools/ImportFile/parser/CloudOptimizedGeoTIFFParser.class.js +++ b/src/modules/menu/components/advancedTools/ImportFile/parser/CloudOptimizedGeoTIFFParser.class.js @@ -1,6 +1,7 @@ import { fromBlob, fromUrl } from 'geotiff' import CloudOptimizedGeoTIFFLayer from '@/api/layers/CloudOptimizedGeoTIFFLayer.class' +import InvalidFileContentError from '@/modules/menu/components/advancedTools/ImportFile/parser/errors/InvalidFileContentError.error' import OutOfBoundsError from '@/modules/menu/components/advancedTools/ImportFile/parser/errors/OutOfBoundsError.error' import UnknownProjectionError from '@/modules/menu/components/advancedTools/ImportFile/parser/errors/UnknownProjectionError.error' import FileParser from '@/modules/menu/components/advancedTools/ImportFile/parser/FileParser.class' @@ -11,6 +12,7 @@ export class CloudOptimizedGeoTIFFParser extends FileParser { constructor() { super({ fileExtensions: ['.tif', '.tiff'], + fileTypeLittleEndianSignature: [0x49, 0x49, 0x2a, 0x00], fileContentTypes: [ 'image/tiff', 'image/tiff;subtype=geotiff', @@ -18,12 +20,21 @@ export class CloudOptimizedGeoTIFFParser extends FileParser { 'application=geotiff; profile=cloud-optimized', 'image/tiff; application=geotiff; profile=cloud-optimized', ], + // loading an entire COG is asking for memory crashes, some can weight more than a terabyte + shouldLoadOnlineContent: false, + allowServiceProxy: false, }) } - async parseCOGLayer(fileSource, geoTIFFInstance, currentProjection) { + async parseFileContent(fileContent, fileSource, currentProjection) { + let geoTIFFInstance + if (this.isLocalFile(fileSource)) { + geoTIFFInstance = await fromBlob(fileSource) + } else { + geoTIFFInstance = await fromUrl(fileSource) + } if (!geoTIFFInstance) { - throw false + throw new InvalidFileContentError('Could not parse COG from file source') } const firstImage = await geoTIFFInstance.getImage() const imageGeoKey = firstImage.getGeoKeys()?.ProjectedCSTypeGeoKey @@ -54,12 +65,4 @@ export class CloudOptimizedGeoTIFFParser extends FileParser { extent: flattenExtent(intersection), }) } - - async parseUrl(fileUrl, currentProjection, options) { - return this.parseCOGLayer(fileUrl, await fromUrl(fileUrl), currentProjection, options) - } - - async parseLocalFile(file, currentProjection) { - return this.parseCOGLayer(file, await fromBlob(file), currentProjection) - } } diff --git a/src/modules/menu/components/advancedTools/ImportFile/parser/FileParser.class.js b/src/modules/menu/components/advancedTools/ImportFile/parser/FileParser.class.js index aee0d1689..a93be2232 100644 --- a/src/modules/menu/components/advancedTools/ImportFile/parser/FileParser.class.js +++ b/src/modules/menu/components/advancedTools/ImportFile/parser/FileParser.class.js @@ -1,7 +1,5 @@ -import axios from 'axios' - -import { getContentThroughServiceProxy } from '@/api/file-proxy.api' -import { getFileFromUrl } from '@/api/files.api' +import { getFileContentThroughServiceProxy } from '@/api/file-proxy.api' +import { checkOnlineFileCompliance, getFileContentFromUrl } from '@/api/files.api' import log from '@/utils/logging' /** @@ -29,21 +27,31 @@ export default class FileParser { * @param {String[]} config.fileContentTypes Which MIME types are typically associated with this * file type. Will be used with one HTTP HEAD request on the file URL (if online file) to * assert if this parser is the valid candidate for the file. + * @param {Boolean} [config.shouldLoadOnlineContent=true] Flag telling if remote content should + * be loaded by this parser to properly parse it. If set to false, no GET (or proxy) request + * will be fired on the file URL. Default is `true` * @param {ValidateFileContent} [config.validateFileContent=null] Function receiving the content * from a file (as ArrayBuffer), and assessing if it is a match for this parser. Default is * `null` + * @param {Boolean} [config.allowServiceProxy=false] Flag telling if the content of the file can + * be requested through service-proxy in case the server hosting the file doesn't support + * CORS. Default is `false` */ constructor(config = {}) { const { fileTypeLittleEndianSignature = [], fileExtensions = [], fileContentTypes = [], + shouldLoadOnlineContent = true, validateFileContent = null, + allowServiceProxy = false, } = config this.fileTypeLittleEndianSignature = fileTypeLittleEndianSignature this.fileExtensions = fileExtensions this.fileContentTypes = fileContentTypes + this.shouldLoadOnlineContent = shouldLoadOnlineContent this.validateFileContent = validateFileContent + this.allowServiceProxy = allowServiceProxy } /** @@ -88,7 +96,7 @@ export default class FileParser { * @returns {Promise} */ async parseLocalFile(file, currentProjection) { - return this.parseFileContent(await file.arrayBuffer(), file.name, currentProjection) + return this.parseFileContent(await file.arrayBuffer(), file, currentProjection) } /** @@ -108,21 +116,19 @@ export default class FileParser { * * @param {String} fileUrl * @param {Object} options - * @param {Boolean} [options.allowServiceProxy] Flag telling if the resource could be accessed - * with the use of service-proxy if another means do not work (MIME type parsing or little - * endian signature detection) - * @param {String} [options.mimeType] MIME type (from a Content-Type HTTP header) gathered from - * a previous HEAD request (so no need to re-run a HEAD request to get this info) + * @param {OnlineFileCompliance} [options.fileCompliance=null] Compliance check results that + * were previously gathered for this file. If not given, this function will run its own + * compliance checks. Default is `null` * @param {ArrayBuffer} [options.loadedContent] File content already loaded (most likely by * service-proxy). When given, no other request will be made on the file source, but this * content will be used instead. * @returns {Promise} */ async isOnlineFileParsingPossible(fileUrl, options = {}) { - const { allowServiceProxy = false, mimeType = null, loadedContent = null } = options + const { fileCompliance = null, loadedContent = null } = options - if (mimeType && !loadedContent) { - return this.fileContentTypes.includes(mimeType) + if (fileCompliance && this.fileContentTypes.includes(fileCompliance.mimeType)) { + return true } if (loadedContent) { @@ -134,25 +140,39 @@ export default class FileParser { fileUrl, this.constructor.name ) - // HEAD request wasn't run yet (we are not coming from the main parseLayerFromFile function) + // HEAD/GET request wasn't run yet (we are not coming from the main parseLayerFromFile function) // so we have to run all the requests ourselves try { - const headResponse = await axios.head(fileUrl) - return this.fileContentTypes.includes(headResponse.headers.get('content-type')) - } catch (error) { - if (allowServiceProxy) { - log.debug( - `[FileParser][${this.constructor.name}] could not access resource through HEAD request, trying through service-proxy`, - fileUrl - ) - return this.isFileContentValid(await getContentThroughServiceProxy(fileUrl)) - } else { - log.debug( - `[FileParser][${this.constructor.name}] HEAD request failed, but service-proxy is not allowed for file, could not parse`, - fileUrl, - this.constructor.name - ) + const { mimeType, supportsCORS, supportsHTTPS } = + await checkOnlineFileCompliance(fileUrl) + // if a MIME type match is found, then all good + if (this.fileContentTypes.includes(mimeType)) { + return true } + // if no MIME type match and file content can be checked, we load it now + if (this.shouldLoadOnlineContent) { + try { + let loadedContent = null + if (supportsCORS && supportsHTTPS) { + loadedContent = await getFileContentFromUrl(fileUrl) + } else { + loadedContent = await getFileContentThroughServiceProxy(fileUrl) + } + return loadedContent && this.isFileContentValid(loadedContent) + } catch (error) { + log.error( + `[FileParser][${this.constructor.name}] Could not load file content for`, + fileUrl, + error + ) + } + } + } catch (error) { + log.warn( + `[FileParser][${this.constructor.name}] HEAD request failed, could not parse`, + fileUrl, + this.constructor.name + ) } return false } @@ -160,7 +180,7 @@ export default class FileParser { /** * @abstract * @param {ArrayBuffer} fileContent - * @param {String} fileSource + * @param {File | String} fileSource * @param {CoordinateSystem} currentProjection * @returns {Promise} */ @@ -174,25 +194,40 @@ export default class FileParser { * against the current projection (and raise OutOfBoundError in case no mutual data is * available) * @param {Object} options - * @param {Number} [options.timeout] The timeout length (in milliseconds) to use when requesting - * online file + * @param {ArrayBuffer} [options.loadedContent] File content already loaded (most likely by + * service-proxy). When given, no other request will be made on the file source, but this + * content will be used instead. + * @param {OnlineFileCompliance} [options.fileCompliance=null] Compliance check results that + * were previously gathered for this file. If not given, this function will run its own + * compliance checks. Default is `null` * @returns {Promise} */ async parseUrl(fileUrl, currentProjection, options = {}) { - const { loadedContent = null } = options + const { loadedContent = null, fileCompliance = null } = options if (loadedContent) { log.debug( `[FileParser][${this.constructor.name}] preloaded content detected, won't create new requests` ) return await this.parseFileContent(loadedContent, fileUrl, currentProjection) + } else if (this.shouldLoadOnlineContent) { + // no preloaded content, we load the file ourselves + // checking for CORS/HTTPS support + const { supportsCORS, supportsHTTPS } = + fileCompliance ?? (await checkOnlineFileCompliance(fileUrl)) + let fileContent = null + if (supportsCORS && supportsHTTPS) { + fileContent = await getFileContentFromUrl(fileUrl) + } else if (this.allowServiceProxy) { + fileContent = await getFileContentThroughServiceProxy(fileUrl) + } else { + throw new Error( + `[FileParser][${this.constructor.name}] could not load content for file ${fileUrl}` + ) + } + + return await this.parseFileContent(fileContent, fileUrl, currentProjection) } - // no preloaded content, we load the file itself - const fileContent = await getFileFromUrl(fileUrl, { - ...options, - // Reading zip archive as text is asking for trouble therefore we use ArrayBuffer (for KMZ) - responseType: 'arraybuffer', - }) - return await this.parseFileContent(fileContent.data, fileUrl, currentProjection) + return await this.parseFileContent(null, fileUrl, currentProjection) } /** @@ -204,11 +239,9 @@ export default class FileParser { * @param {Object} [options] * @param {Number} [options.timeout] The timeout length (in milliseconds) to use when requesting * online file - * @param {Boolean} [options.allowServiceProxy=false] Flag telling if the resource could be - * accessed with the use of service-proxy if another means do not work (MIME type parsing or - * little endian signature detection). Default is `false` - * @param {String} [options.mimeType] MIME type (from a Content-Type HTTP header) gathered from - * a previous HEAD request (so no need to re-run a HEAD request to get this info) + * @param {OnlineFileCompliance} [options.fileCompliance=null] Compliance check results that + * were previously gathered for this file. If not given, this function will run its own + * compliance checks. Default is `null` * @param {ArrayBuffer} [options.loadedContent] File content already loaded (most likely by * service-proxy). When given, no other request will be made on the file source, but this * content will be used instead. diff --git a/src/modules/menu/components/advancedTools/ImportFile/parser/GPXParser.class.js b/src/modules/menu/components/advancedTools/ImportFile/parser/GPXParser.class.js index 8fec8f292..a79193043 100644 --- a/src/modules/menu/components/advancedTools/ImportFile/parser/GPXParser.class.js +++ b/src/modules/menu/components/advancedTools/ImportFile/parser/GPXParser.class.js @@ -28,6 +28,7 @@ export default class GPXParser extends FileParser { fileExtensions: ['.gpx'], fileContentTypes: ['application/gpx+xml', 'application/xml', 'text/xml'], validateFileContent: isGpx, + allowServiceProxy: true, }) } @@ -49,7 +50,7 @@ export default class GPXParser extends FileParser { throw new OutOfBoundsError(`GPX is out of bounds of current projection: ${extent}`) } return new GPXLayer({ - gpxFileUrl: fileSource, + gpxFileUrl: this.isLocalFile(fileSource) ? fileSource.name : fileSource, visible: true, opacity: 1.0, gpxData: gpxAsText, diff --git a/src/modules/menu/components/advancedTools/ImportFile/parser/KMLParser.class.js b/src/modules/menu/components/advancedTools/ImportFile/parser/KMLParser.class.js index 73ec94b66..9288aa675 100644 --- a/src/modules/menu/components/advancedTools/ImportFile/parser/KMLParser.class.js +++ b/src/modules/menu/components/advancedTools/ImportFile/parser/KMLParser.class.js @@ -30,12 +30,13 @@ export class KMLParser extends FileParser { 'text/xml', ], validateFileContent: isKml, + allowServiceProxy: true, }) } /** * @param {ArrayBuffer} fileContent - * @param fileSource + * @param {String | File} fileSource * @param currentProjection * @param {Map} [linkFiles] Used in the context of a KMZ to carry the * embedded files with the layer @@ -59,7 +60,7 @@ export class KMLParser extends FileParser { throw new OutOfBoundsError(`KML is out of bounds of current projection: ${extent}`) } return new KMLLayer({ - kmlFileUrl: fileSource, + kmlFileUrl: this.isLocalFile(fileSource) ? fileSource.name : fileSource, visible: true, opacity: 1.0, adminId: null, diff --git a/src/modules/menu/components/advancedTools/ImportFile/parser/KMZParser.class.js b/src/modules/menu/components/advancedTools/ImportFile/parser/KMZParser.class.js index de78adfa1..7294f02ba 100644 --- a/src/modules/menu/components/advancedTools/ImportFile/parser/KMZParser.class.js +++ b/src/modules/menu/components/advancedTools/ImportFile/parser/KMZParser.class.js @@ -36,12 +36,15 @@ export default class KMZParser extends FileParser { /** * @param {ArrayBuffer} data - * @param {String} fileSource + * @param {String | File} fileSource * @param {CoordinateSystem} currentProjection * @returns {Promise} */ async parseFileContent(data, fileSource, currentProjection) { - const kmz = await unzipKmz(data, fileSource) + const kmz = await unzipKmz( + data, + this.isLocalFile(fileSource) ? fileSource.name : fileSource + ) return kmlParser.parseFileContent(kmz.kml, kmz.name, currentProjection, kmz.files) } } diff --git a/src/modules/menu/components/advancedTools/ImportFile/parser/index.js b/src/modules/menu/components/advancedTools/ImportFile/parser/index.js index 8c0654f38..daf39169e 100644 --- a/src/modules/menu/components/advancedTools/ImportFile/parser/index.js +++ b/src/modules/menu/components/advancedTools/ImportFile/parser/index.js @@ -1,5 +1,5 @@ -import { getContentThroughServiceProxy } from '@/api/file-proxy.api' -import { getFileFromUrl, getFileMimeType } from '@/api/files.api' +import { getFileContentThroughServiceProxy } from '@/api/file-proxy.api' +import { checkOnlineFileCompliance, getFileContentFromUrl } from '@/api/files.api' import AbstractLayer from '@/api/layers/AbstractLayer.class' import { CloudOptimizedGeoTIFFParser } from '@/modules/menu/components/advancedTools/ImportFile/parser/CloudOptimizedGeoTIFFParser.class' import GPXParser from '@/modules/menu/components/advancedTools/ImportFile/parser/GPXParser.class' @@ -19,8 +19,7 @@ const allParsers = [ * @param {File | String} config.fileSource * @param {CoordinateSystem} config.currentProjection * @param {Object} [options] - * @param {Boolean} [options.allowServiceProxy=false] Default is `false` - * @param {String} [options.mimeType] + * @param {OnlineFileCompliance} [options.fileCompliance] * @param {ArrayBuffer} [options.loadedContent] * @returns {Promise} */ @@ -44,124 +43,76 @@ async function parseAll(config, options) { } /** - * Will (attempt to) load the MIME type of the file (through a HEAD request) and then decide if it - * requires service-proxy to get the content (HEAD request failed) or if the content should be - * loaded with a simple GET request. - * - * If nothing could get loaded (HTTP 4xx or 5xx for both request) it will return `null` for both - * mimeType and loadedContent (no error will be thrown) - * - * @param {String} fileUrl - * @param {Object} [options] - * @param {Boolean} [options.allowServiceProxy=false] Flag telling the function if it can use - * service-proxy in case a HEAD feature failed (potential CORS issue) on the file URL. Only use it - * for file format that are meant for non-technical users (KML/GPX,etc...). Default is `false` - * @returns {Promise<{ mimeType: [String, null]; loadedContent: [null, ArrayBuffer] }>} + * @param {File | String} fileSource + * @param {CoordinateSystem} currentProjection Can be used to check bounds of parsed file against + * the current projection (and raise OutOfBoundError in case no mutual data is available) + * @returns {Promise} */ -export async function getOnlineFileContent(fileUrl, options) { - const { allowServiceProxy = false } = options - let mimeType = null - let loadedContent = null - try { - mimeType = await getFileMimeType(fileUrl) - log.debug('[FileParser] got MIME type', mimeType, 'for file', fileUrl) - } catch (error) { - log.debug( - '[FileParser][getOnlineFileContent] could not have a HEAD response on', - fileUrl, - 'this file might require service-proxy', - error - ) +export async function parseLayerFromFile(fileSource, currentProjection) { + // if local file, just parse it + if (fileSource instanceof File) { + return await parseAll({ + fileSource, + currentProjection, + }) } - if (!mimeType && allowServiceProxy) { - try { - loadedContent = await getContentThroughServiceProxy(fileUrl) - } catch (error) { - log.error( - '[FileParser][getOnlineFileContent] could not get content of file', - fileUrl, - 'through service-proxy', - error + + // online file, we start by getting its MIME type and other compliance information (CORS, HTTPS) + const fileComplianceCheck = await checkOnlineFileCompliance(fileSource) + log.debug( + '[FileParser][parseLayerFromFile] file', + fileSource, + 'has compliance', + fileComplianceCheck + ) + const { mimeType, supportsCORS, supportsHTTPS } = fileComplianceCheck + + if (mimeType) { + // trying to find a matching parser only based on MIME type. This is required for COG, as we + // can't be loading the entire COG data to check against our parsers (some COG can weight in the To) + const parserMatchingMIME = allParsers.find((parser) => + parser.fileContentTypes.includes(mimeType) + ) + if (parserMatchingMIME) { + log.debug( + '[FileParser][parseLayerFromFile] parser found for MIME type', + mimeType, + parserMatchingMIME ) - } - } - if (!loadedContent) { - try { - const fileContentRequest = await getFileFromUrl(fileUrl, { - responseType: 'arraybuffer', + return parserMatchingMIME.parseUrl(fileSource, currentProjection, { + fileCompliance: fileComplianceCheck, }) - loadedContent = fileContentRequest.data - } catch (error) { - log.error( - '[FileParser][getOnlineFileContent] could not get content for file', - fileUrl, - error - ) } } - return { - loadedContent, - mimeType, - } -} -/** - * @param {File | String} fileSource - * @param {CoordinateSystem} currentProjection Can be used to check bounds of parsed file against - * the current projection (and raise OutOfBoundError in case no mutual data is available) - * @param {Object} [options] - * @param {Number} [options.timeout] The timeout length (in milliseconds) to use when requesting - * online file - * @returns {Promise} - */ -export async function parseLayerFromFile(fileSource, currentProjection, options = {}) { - const isLocalFile = fileSource instanceof File - if (!isLocalFile) { - // first pass without using service-proxy - const { mimeType, loadedContent } = await getOnlineFileContent(fileSource, { - allowServiceProxy: false, - }) - const resultsWithoutServiceProxy = await parseAll( + // no MIME type match, getting file content and trying to find a parser that can handle it + try { + log.debug( + '[FileParser][parseLayerFromFile] no MIME type match, loading file content for', + fileSource + ) + let loadedContent = null + if (supportsCORS && supportsHTTPS) { + loadedContent = await getFileContentFromUrl(fileSource) + } else { + loadedContent = await getFileContentThroughServiceProxy(fileSource) + } + return await parseAll( { fileSource, currentProjection, }, { - ...options, - mimeType, + fileCompliance: fileComplianceCheck, loadedContent, - allowServiceProxy: false, } ) - if (resultsWithoutServiceProxy) { - return resultsWithoutServiceProxy - } - // getting content through service-proxy and running another pass of parsing - try { - const proxyfiedContent = await getContentThroughServiceProxy(fileSource) - return await parseAll( - { - fileSource, - currentProjection, - }, - { - ...options, - mimeType, - loadedContent: proxyfiedContent, - allowServiceProxy: false, - } - ) - } catch (error) { - log.error( - '[FileParser][parseLayerFromFile] could not get content through service-proxy for file', - fileSource, - error - ) - throw error - } + } catch (error) { + log.error( + '[FileParser][parseLayerFromFile] could not get content for file', + fileSource, + error + ) + throw error } - return await parseAll({ - fileSource, - currentProjection, - }) } diff --git a/src/store/plugins/load-cog-extent.plugin.js b/src/store/plugins/load-cog-extent.plugin.js index 9b7ec6801..e70504f4d 100644 --- a/src/store/plugins/load-cog-extent.plugin.js +++ b/src/store/plugins/load-cog-extent.plugin.js @@ -6,15 +6,10 @@ import { CloudOptimizedGeoTIFFParser } from '@/modules/menu/components/advancedT const cogParser = new CloudOptimizedGeoTIFFParser() async function loadExtentAndUpdateLayer(store, layer) { - const layerWithExtent = await cogParser.parse( - { - fileSource: layer.fileSource, - currentProjection: toValue(store.state.position.projection), - }, - { - allowServiceProxy: false, - } - ) + const layerWithExtent = await cogParser.parse({ + fileSource: layer.fileSource, + currentProjection: toValue(store.state.position.projection), + }) store.dispatch('updateLayer', { layerId: layer.id, values: { diff --git a/src/store/plugins/load-gpx-data.plugin.js b/src/store/plugins/load-gpx-data.plugin.js index b1b7e1b52..0c7dddb21 100644 --- a/src/store/plugins/load-gpx-data.plugin.js +++ b/src/store/plugins/load-gpx-data.plugin.js @@ -20,15 +20,10 @@ const gpxParser = new GPXParser() async function loadGpx(store, gpxLayer) { log.debug(`Loading data for added GPX layer`, gpxLayer) try { - const updatedLayer = await gpxParser.parse( - { - fileSource: gpxLayer.gpxFileUrl, - currentProjection: store.state.position.projection, - }, - { - allowServiceProxy: true, - } - ) + const updatedLayer = await gpxParser.parse({ + fileSource: gpxLayer.gpxFileUrl, + currentProjection: store.state.position.projection, + }) store.dispatch('updateLayer', { layerId: updatedLayer.id, values: updatedLayer, diff --git a/src/store/plugins/load-kml-kmz-data.plugin.js b/src/store/plugins/load-kml-kmz-data.plugin.js index bdd9a37a1..f1ba496f8 100644 --- a/src/store/plugins/load-kml-kmz-data.plugin.js +++ b/src/store/plugins/load-kml-kmz-data.plugin.js @@ -3,9 +3,9 @@ * it here */ -import { loadKmlMetadata } from '@/api/files.api' +import { getFileContentThroughServiceProxy } from '@/api/file-proxy.api' +import { checkOnlineFileCompliance, getFileContentFromUrl, loadKmlMetadata } from '@/api/files.api' import KMLLayer from '@/api/layers/KMLLayer.class' -import { getOnlineFileContent } from '@/modules/menu/components/advancedTools/ImportFile/parser' import generateErrorMessageFromErrorType from '@/modules/menu/components/advancedTools/ImportFile/parser/errors/generateErrorMessageFromErrorType.utils' import { KMLParser } from '@/modules/menu/components/advancedTools/ImportFile/parser/KMLParser.class' import KMZParser from '@/modules/menu/components/advancedTools/ImportFile/parser/KMZParser.class' @@ -45,11 +45,25 @@ async function loadMetadata(store, kmlLayer) { */ async function loadData(store, kmlLayer) { log.debug(`Loading data for added KML layer`, kmlLayer) - // to avoid having 2 HEAD and 2 GET request in case the file is a KML, we load this data here (instead of letting each file parser load it for itself) - const { mimeType, loadedContent } = await getOnlineFileContent(kmlLayer.kmlFileUrl, { - responseType: 'arraybuffer', - }) + // to avoid having 2 HEAD and 2 GET request in case the file is a KML, we load this data here (instead of letting each file parser load it for itself) + const { mimeType, supportsCORS, supportsHTTPS } = await checkOnlineFileCompliance( + kmlLayer.kmlFileUrl + ) + let loadedContent = null + try { + if (supportsCORS && supportsHTTPS) { + loadedContent = await getFileContentFromUrl(kmlLayer.kmlFileUrl) + } else if (mimeType) { + loadedContent = await getFileContentThroughServiceProxy(kmlLayer.kmlFileUrl) + } + } catch (error) { + log.error( + '[load-kml-kmz-data] error while loading file content for', + kmlLayer.kmlFileUrl, + error + ) + } if (!mimeType && !loadedContent) { log.error('[load-kml-kmz-data] could not get content for KML', kmlLayer.kmlFileUrl) store.dispatch('addLayerError', { @@ -69,7 +83,6 @@ async function loadData(store, kmlLayer) { currentProjection: store.state.position.projection, }, { - allowServiceProxy: false, mimeType, loadedContent, } @@ -92,7 +105,6 @@ async function loadData(store, kmlLayer) { currentProjection: store.state.position.projection, }, { - allowServiceProxy: false, mimeType, loadedContent, } diff --git a/tests/cypress/tests-e2e/print.cy.js b/tests/cypress/tests-e2e/print.cy.js index 21c024e90..1c58b6101 100644 --- a/tests/cypress/tests-e2e/print.cy.js +++ b/tests/cypress/tests-e2e/print.cy.js @@ -31,10 +31,6 @@ describe('Testing print', () => { }).as('printRequest') } - function interceptKml(fixture) { - cy.intercept('GET', '**/**.kml', { fixture }).as('kmlRequest') - } - function interceptPrintStatus() { cy.intercept('GET', '**/status/**', (req) => { req.reply({ @@ -179,7 +175,11 @@ describe('Testing print', () => { interceptPrintRequest() interceptPrintStatus() interceptDownloadReport() - interceptKml(kmlFixture) + + cy.intercept('HEAD', '**/**.kml', { + headers: { 'Content-Type': 'application/vnd.google-earth.kml+xml' }, + }).as('kmlHeadRequest') + cy.intercept('GET', '**/**.kml', { fixture: kmlFixture }).as('kmlGetRequest') cy.goToMapView( { @@ -188,7 +188,7 @@ describe('Testing print', () => { }, true ) - cy.wait('@kmlRequest') + cy.wait(['@kmlHeadRequest', '@kmlGetRequest']) cy.readStoreValue('state.layers.activeLayers').should('have.length', 1) cy.openMenuIfMobile()