Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2023.02.xx] #9590: COG download metadata by default with abort fetch (#9687) #9713

Merged
merged 1 commit into from
Nov 20, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions web/client/actions/__tests__/catalog-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,13 @@ describe('Test correctness of the catalog actions', () => {
expect(retval).toExist();
expect(retval.type).toBe(ADD_SERVICE);
});
it('addService with options', () => {
const options = {"test": "1"};
var retval = addService(options);
expect(retval).toExist();
expect(retval.type).toBe(ADD_SERVICE);
expect(retval.options).toEqual(options);
});
it('addCatalogService', () => {
var retval = addCatalogService(service);

Expand Down
5 changes: 3 additions & 2 deletions web/client/actions/catalog.js
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,10 @@ export function changeUrl(url) {
url
};
}
export function addService() {
export function addService(options) {
return {
type: ADD_SERVICE
type: ADD_SERVICE,
options
};
}
export function addCatalogService(service) {
Expand Down
56 changes: 44 additions & 12 deletions web/client/api/catalog/COG.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@

import get from 'lodash/get';
import isEmpty from 'lodash/isEmpty';
import isNil from 'lodash/isNil';
import { Observable } from 'rxjs';
import { fromUrl } from 'geotiff';
import { fromUrl as fromGeotiffUrl } from 'geotiff';

import { isValidURL } from '../../utils/URLUtils';
import ConfigUtils from '../../utils/ConfigUtils';
Expand Down Expand Up @@ -57,8 +58,25 @@ export const getProjectionFromGeoKeys = (image) => {

return null;
};
const abortError = (reject) => reject(new DOMException("Aborted", "AbortError"));
/**
* fromUrl with abort fetching of data and data slices
* Note: The abort action will not cancel data fetch request but just the promise,
* because of the issue in https://github.com/geotiffjs/geotiff.js/issues/408
*/
const fromUrl = (url, signal) => {
if (signal?.aborted) {
return abortError(Promise.reject);
}
return new Promise((resolve, reject) => {
signal?.addEventListener("abort", () => abortError(reject));
return fromGeotiffUrl(url)
.then((image)=> image.getImage()) // Fetch and read first image to get medatadata of the tif
.then((image) => resolve(image))
.catch(()=> abortError(reject));
});
};
let capabilitiesCache = {};

export const getRecords = (_url, startPosition, maxRecords, text, info = {}) => {
const service = get(info, 'options.service');
let layers = [];
Expand All @@ -73,29 +91,43 @@ export const getRecords = (_url, startPosition, maxRecords, text, info = {}) =>
sources: [{url}],
options: service.options || {}
};
if (service.fetchMetadata) {
const controller = get(info, 'options.controller');
const isSave = get(info, 'options.save', false);
// Fetch metadata only on saving the service (skip on search)
if ((isNil(service.fetchMetadata) || service.fetchMetadata) && isSave) {
const cached = capabilitiesCache[url];
if (cached && new Date().getTime() < cached.timestamp + (ConfigUtils.getConfigProp('cacheExpire') || 60) * 1000) {
return {...cached.data};
}
return fromUrl(url)
.then(geotiff => geotiff.getImage())
return fromUrl(url, controller?.signal)
.then(image => {
const crs = getProjectionFromGeoKeys(image);
const extent = image.getBoundingBox();
const isProjectionDefined = isProjectionAvailable(crs);
layer = {
...layer,
sourceMetadata: {
crs,
extent: extent,
width: image.getWidth(),
height: image.getHeight(),
tileWidth: image.getTileWidth(),
tileHeight: image.getTileHeight(),
origin: image.getOrigin(),
resolution: image.getResolution()
},
// skip adding bbox when geokeys or extent is empty
...(!isEmpty(extent) && !isEmpty(crs) && isProjectionDefined && {
...(!isEmpty(extent) && !isEmpty(crs) && {
bbox: {
crs,
bounds: {
minx: extent[0],
miny: extent[1],
maxx: extent[2],
maxy: extent[3]
}
...(isProjectionDefined && {
bounds: {
minx: extent[0],
miny: extent[1],
maxx: extent[2],
maxy: extent[3]
}}
)
}
})
};
Expand Down
30 changes: 23 additions & 7 deletions web/client/components/catalog/CatalogServiceEditor.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,20 @@ import Message from "../I18N/Message";
import AdvancedSettings from './editor/AdvancedSettings';
import MainForm from './editor/MainForm';

export default ({
const withAbort = (Component) => {
return (props) => {
const [abortController, setAbortController] = useState(null);
const onSave = () => {
// Currently abort request on saving is applicable only for COG service
const controller = props.format === 'cog' ? new AbortController() : null;
setAbortController(controller);
return props.onAddService({save: true, controller});
};
const onCancel = () => abortController && props.saving ? abortController?.abort() : props.onChangeCatalogMode("view");
return <Component {...props} onCancel={onCancel} onSaveService={onSave} disabled={!abortController && props.saving} />;
};
};
const CatalogServiceEditor = ({
service = {
title: "",
type: "wms",
Expand All @@ -39,9 +52,9 @@ export default ({
onChangeServiceProperty = () => {},
onToggleTemplate = () => {},
onToggleThumbnail = () => {},
onAddService = () => {},
onDeleteService = () => {},
onChangeCatalogMode = () => {},
onCancel = () => {},
onSaveService = () => {},
onFormatOptionsFetch = () => {},
selectedService,
isLocalizedLayerStylesEnabled,
Expand All @@ -50,7 +63,8 @@ export default ({
layerOptions = {},
infoFormatOptions,
services,
autoSetVisibilityLimits = false
autoSetVisibilityLimits = false,
disabled
}) => {
const [valid, setValid] = useState(true);
return (<BorderLayout
Expand Down Expand Up @@ -92,21 +106,23 @@ export default ({
/>
<FormGroup controlId="buttons" key="buttons">
<Col xs={12}>
<Button style={buttonStyle} disabled={saving || !valid} onClick={() => onAddService()} key="catalog_add_service_button">
<Button style={buttonStyle} disabled={saving || !valid} onClick={onSaveService} key="catalog_add_service_button">
{saving ? <Loader size={12} style={{display: 'inline-block'}} /> : null}
<Message msgId="save" />
</Button>
{service && !service.isNew
? <Button style={buttonStyle} onClick={() => onDeleteService(service, services)} key="catalog_delete_service_button">
? <Button style={buttonStyle} disabled={saving} onClick={() => onDeleteService(service, services)} key="catalog_delete_service_button">
<Message msgId="catalog.delete" />
</Button>
: null
}
<Button style={buttonStyle} disabled={saving} onClick={() => onChangeCatalogMode("view")} key="catalog_back_view_button">
<Button style={buttonStyle} disabled={disabled} onClick={onCancel} key="catalog_back_view_button">
<Message msgId="cancel" />
</Button>
</Col>
</FormGroup>
</Form>
</BorderLayout>);
};

export default withAbort(CatalogServiceEditor);
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import React from 'react';
import ReactDOM from 'react-dom';
import expect from 'expect';
import TestUtils from 'react-dom/test-utils';
import CatalogServiceEditor from '../CatalogServiceEditor';
import {defaultPlaceholder} from "../editor/MainFormUtils";

Expand Down Expand Up @@ -149,4 +150,93 @@ describe('Test CatalogServiceEditor', () => {
let placeholder = defaultPlaceholder(service);
expect(placeholder).toBe("e.g. https://mydomain.com/geoserver/wms");
});
it('test save and delete button when saving', () => {
ReactDOM.render(<CatalogServiceEditor
service={givenWmsService}
layerOptions={{tileSize: 256}}
saving
/>, document.getElementById("container"));
let buttons = document.querySelectorAll('.form-group button');
let saveBtn; let deleteBtn;
buttons.forEach(btn => {if (btn.textContent === 'save') saveBtn = btn;});
buttons.forEach(btn => {if (btn.textContent === 'catalog.delete') deleteBtn = btn;});
expect(saveBtn).toBeTruthy();
expect(deleteBtn).toBeTruthy();
expect(saveBtn.classList.contains("disabled")).toBeTruthy();
expect(deleteBtn.classList.contains("disabled")).toBeTruthy();
});
it('test saving service for COG type', () => {
const actions = {
onAddService: () => {}
};
const spyOnAdd = expect.spyOn(actions, 'onAddService');
ReactDOM.render(<CatalogServiceEditor
format="cog"
onAddService={actions.onAddService}
/>, document.getElementById("container"));
let buttons = document.querySelectorAll('.form-group button');
let saveBtn;
buttons.forEach(btn => {if (btn.textContent === 'save') saveBtn = btn;});
expect(saveBtn).toBeTruthy();
TestUtils.Simulate.click(saveBtn);
expect(spyOnAdd).toHaveBeenCalled();
let arg = spyOnAdd.calls[0].arguments[0];
expect(arg.save).toBe(true);
expect(arg.controller).toBeTruthy();

ReactDOM.render(<CatalogServiceEditor
format="csw"
onAddService={actions.onAddService}
/>, document.getElementById("container"));
buttons = document.querySelectorAll('.form-group button');
buttons.forEach(btn => {if (btn.textContent === 'save') saveBtn = btn;});
expect(saveBtn).toBeTruthy();
TestUtils.Simulate.click(saveBtn);
expect(spyOnAdd).toHaveBeenCalled();
arg = spyOnAdd.calls[1].arguments[0];
expect(arg.save).toBeTruthy();
expect(arg.controller).toBeFalsy();
});
it('test cancel service', () => {
const actions = {
onChangeCatalogMode: () => {},
onAddService: () => {}
};
const spyOnCancel = expect.spyOn(actions, 'onChangeCatalogMode');
ReactDOM.render(<CatalogServiceEditor
format="csw"
onChangeCatalogMode={actions.onChangeCatalogMode}
/>, document.getElementById("container"));
let buttons = document.querySelectorAll('.form-group button');
let cancelBtn;
buttons.forEach(btn => {if (btn.textContent === 'cancel') cancelBtn = btn;});
expect(cancelBtn).toBeTruthy();
TestUtils.Simulate.click(cancelBtn);
expect(spyOnCancel).toHaveBeenCalled();
let arg = spyOnCancel.calls[0].arguments[0];
expect(arg).toBe('view');

const spyOnAdd = expect.spyOn(actions, 'onAddService');
ReactDOM.render(<CatalogServiceEditor
format="cog"
onChangeCatalogMode={actions.onChangeCatalogMode}
onAddService={actions.onAddService}
/>, document.getElementById("container"));
buttons = document.querySelectorAll('.form-group button');
let saveBtn;
buttons.forEach(btn => {if (btn.textContent === 'save') saveBtn = btn;});
TestUtils.Simulate.click(saveBtn);
expect(spyOnAdd).toHaveBeenCalled();

ReactDOM.render(<CatalogServiceEditor
format="cog"
saving
onChangeCatalogMode={actions.onChangeCatalogMode}
onAddService={actions.onAddService}
/>, document.getElementById("container"));
buttons = document.querySelectorAll('.form-group button');
buttons.forEach(btn => {if (btn.textContent === 'cancel') cancelBtn = btn;});
TestUtils.Simulate.click(cancelBtn);
expect(spyOnCancel.calls[1]).toBeFalsy();
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ export default ({
<Col xs={12}>
<Checkbox
onChange={(e) => onChangeServiceProperty("fetchMetadata", e.target.checked)}
checked={!isNil(service.fetchMetadata) ? service.fetchMetadata : false}>
checked={!isNil(service.fetchMetadata) ? service.fetchMetadata : true}>
<Message msgId="catalog.fetchMetadata.label" />&nbsp;<InfoPopover text={<Message msgId="catalog.fetchMetadata.tooltip" />} />
</Checkbox>
</Col>
Expand Down
4 changes: 2 additions & 2 deletions web/client/epics/catalog.js
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ export default (API) => ({
*/
newCatalogServiceAdded: (action$, store) =>
action$.ofType(ADD_SERVICE)
.switchMap(() => {
.switchMap(({options} = {}) => {
const state = store.getState();
const newService = newServiceSelector(state);
const maxRecords = pageSizeSelector(state);
Expand All @@ -310,7 +310,7 @@ export default (API) => ({
startPosition: 1,
maxRecords,
text: "",
options: {service, isNewService: true}
options: {service, isNewService: true, ...options}
})
);
})
Expand Down
2 changes: 1 addition & 1 deletion web/client/translations/data.de-DE.json
Original file line number Diff line number Diff line change
Expand Up @@ -1534,7 +1534,7 @@
"tooltip": "Fügen Sie der Karte Ebenen hinzu",
"autoload": "Suche in Dienstauswahl",
"fetchMetadata": {
"label": "Laden Sie Dateimetadaten bei der Suche herunter",
"label": "Dateimetadaten beim Speichern herunterladen",
"tooltip": "Diese Option ruft Metadaten ab, um das Zoomen auf Ebene zu unterstützen. Es kann den Suchvorgang verlangsamen, wenn die Bilder zu groß oder zu viele sind."
},
"clearValueText": "Auswahl aufheben",
Expand Down
2 changes: 1 addition & 1 deletion web/client/translations/data.en-US.json
Original file line number Diff line number Diff line change
Expand Up @@ -1495,7 +1495,7 @@
"tooltip": "Add layers to the map",
"autoload": "Search on service selection",
"fetchMetadata": {
"label": "Download file metadata on search",
"label": "Download file metadata on save",
"tooltip": "This option will fetch metadata to support the zoom to layer. It may slow down the search operation if the images are too big or too many."
},
"clearValueText": "Clear selection",
Expand Down
2 changes: 1 addition & 1 deletion web/client/translations/data.es-ES.json
Original file line number Diff line number Diff line change
Expand Up @@ -1496,7 +1496,7 @@
"tooltip": "agregar capas al mapa",
"autoload": "Buscar en la selección de servicios",
"fetchMetadata": {
"label": "Descargar metadatos de archivos en la búsqueda",
"label": "Descargar metadatos del archivo al guardar",
"tooltip": "Esta opción recuperará metadatos para admitir el zoom a la capa. Puede ralentizar la operación de búsqueda si las imágenes son demasiado grandes o demasiadas."
},
"clearValueText": "Borrar selección",
Expand Down
2 changes: 1 addition & 1 deletion web/client/translations/data.fr-FR.json
Original file line number Diff line number Diff line change
Expand Up @@ -1496,7 +1496,7 @@
"tooltip": "Ajouter des couches à la carte",
"autoload": "Recherche sur la sélection du service",
"fetchMetadata": {
"label": "Télécharger les métadonnées du fichier lors de la recherche",
"label": "Télécharger les métadonnées du fichier lors de l'enregistrement",
"tooltip": "Cette option récupérera les métadonnées pour prendre en charge le zoom sur la couche. Cela peut ralentir l'opération de recherche si les images sont trop grandes ou trop nombreuses."
},
"clearValueText": "Effacer la sélection",
Expand Down
2 changes: 1 addition & 1 deletion web/client/translations/data.it-IT.json
Original file line number Diff line number Diff line change
Expand Up @@ -1494,7 +1494,7 @@
"title": "Catalogo",
"autoload": "Ricerca alla selezione del servizio",
"fetchMetadata": {
"label": "Scarica i metadati dei file durante la ricerca",
"label": "Scarica i metadati del file al salvataggio",
"tooltip": "Questa opzione recupererà i metadati per supportare lo zoom a livello. Potrebbe rallentare l'operazione di ricerca se le immagini sono troppo grandi o troppe."
},
"clearValueText": "Cancella selezione",
Expand Down
Loading