diff --git a/README.md b/README.md index b30e0e9..388d713 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Create a QGIS project on the fly on the server -* Supported raster formats : asc, tif, tiff +* Supported raster formats : asc, tif, tiff, geotiff, geotif * Supported vector formats : shp, geojson **This project is still in development, the API may change.** @@ -10,11 +10,12 @@ * Parameters : * SERVICE=MAPCOMPOSITION, compulsory * PROJECT, compulsory, path where the project will be written on the file system. - * FILES, compulsory, it's a list of files on the filesystem, separated by a semicolon. + * SOURCES, compulsory, it's a list of layer sources. It can be tile url or QGIS DataSource URI or files on the filesystem, separated by a semicolon. + Especially for QGIS DataSource URI, it must be url quoted twice (first the url, second the whole datasource string). + * FILES, optional, legacy parameter, it's a list of files on the filesystem, separated by a semicolon. It is overriden by SOURCES. * NAMES, compulsory, it's a list of names, separated by a semicolon. It will be used for the legend. Items in this list should match layers in the FILES list. * OVERWRITE, optional, false by default. Boolean if we can overwrite the existing PROJECT above. Values can be '1', 'YES', 'TRUE', 'yes', 'true'. * REMOVEQML, optional, false by default. Boolean if we can remove the QML. The style is already in the QGS file. Values can be '1', 'YES', 'TRUE', 'yes', 'true'. - * BASEMAP, optional, None by default. it's a list of string comprised of a tile url and its service name, separated by a semicolon. Layers need to be stored on the server's filesystem. The project will be created at the specified path above, on the server's filesystem too. @@ -40,6 +41,47 @@ MAP=/destination/project.qgs& REQUEST=GetCapabilities ``` +* Example of using SOURCES with tile URI: + +``` +http://localhost:81/qgis? +SERVICE=MAPCOMPOSITION& +PROJECT=/destination/project.qgs& +SOURCES=type%253Dxyz%2526url%253Dhttp%25253A%2F%2Fa.tile.osm.org%2F%25257Bz%25257D%2F%25257Bx%25257D%2F%25257By%25257D.png;/path/1.shp;/path/2.geojson;/path/3.asc& +NAMES=Basemap;My layer 1;MyLayer 2;Layer 3& +OVERWRITE=true +``` + +In the sample request above, note that the datasource: ```type%253Dxyz%2526url%253Dhttp%25253A%2F%2Fa.tile.osm.org%2F%25257Bz%25257D%2F%25257Bx%25257D%2F%25257By%25257D.png``` were urlquoted twice. +The actual datasource is: ```type=xyz&url=http%3A//a.tile.osm.org/%7Bz%7D/%7Bx%7D/%7By%7D.png``` +Note that the actual url is: ```http://a.tile.osm.org/{z}/{x}/{y}.png``` + +Thus in order to send the request, the url needs to be quoted first before it was inserted into datasource uri (to quote & symbol and = from url). +Then, the datasource needs to be quoted again, because it was sent via GET requests url (to quote & symbol and = from datasource query params). + +As an example of sending your datasource of base layer: + +Quote your tile url: + +``` +>>> tile_url = 'http://a.tile.osm.org/{z}/{x}/{y}.png' +>>> from requests.utils import quote +>>> tile_url = quote(tile_url) +``` + +Then build your data source definition and quote it: + +``` +>>> definition = { +... 'url': tile_url, +... 'type': 'xyz' +... } +>>> datasource = '&'.join(['{key}={value}'.format(key=key,value=value) for key,value in definition.iteritems()]) +'url=http%3A//a.tile.osm.org/%7Bz%7D/%7Bx%7D/%7By%7D.png&type=xyz' +>>> quoted_datasource = quote(datasource) +'url%3Dhttp%253A//a.tile.osm.org/%257Bz%257D/%257Bx%257D/%257By%257D.png%26type%3Dxyz' +``` + ## Todo * Add WCS * Add tests @@ -107,4 +149,4 @@ LAYERS=your_quoted_layers_variable ``` docker pull kartoza/qgis-server:LTR QUERY_STRING="SERVICE=STYLEMANAGER&PROJECT=/usr/src/app/geonode/qgis_layer/small_building.qgs&REQUEST=GetStyle&LAYER=build&NAME=toto" /usr/lib/cgi-bin/qgis_mapserv.fcgi -``` \ No newline at end of file +``` diff --git a/filters/map_composition.py b/filters/map_composition.py index be88bd6..8b57ee9 100644 --- a/filters/map_composition.py +++ b/filters/map_composition.py @@ -26,7 +26,12 @@ QgsMessageLog, QgsVectorLayer, QgsRasterLayer) -from .tools import generate_legend + +from .tools import ( + generate_legend, + validate_source_uri, + is_file_path, + layer_from_source) class MapComposition(QgsServerFilter): @@ -43,11 +48,12 @@ def responseComplete(self): Example : SERVICE=MAPCOMPOSITION& PROJECT=/destination/project.qgs& - FILES=/path/1.shp;/path/2.shp;/path/3.asc& + SOURCES=type=xyz&url=http://tile.osm.org/{z}/{x}/{y}.png?layers=osm; + /path/1.shp;/path/2.shp;/path/3.asc& + FILES={Legacy Name for Sources Parameter} NAMES=Layer 1;Layer 2;Layer 3& REMOVEQML=true& OVERWRITE=true& - BASEMAP='type=xyz&url=http://tile.osm.org/{z}/{x}/{y}.png?layers=osm;osm' """ request = self.serverInterface().requestHandler() params = request.parameterMap() @@ -80,64 +86,78 @@ def responseComplete(self): else: remove_qml = False - if exists(project_path): - if not overwrite: - msg = 'PROJECT is already existing : %s \n' % project_path - request.appendBody(msg) - return - else: - remove(project_path) + if exists(project_path) and overwrite: + # Overwrite means create from scratch again + remove(project_path) - files_parameters = params.get('FILES') - if not files_parameters: - request.appendBody('FILES parameter is missing.\n') + sources_parameters = params.get('SOURCES') + # support legacy params: FILES + if not sources_parameters: + sources_parameters = params.get('FILES') + + if not sources_parameters: + request.appendBody('SOURCES parameter is missing.\n') return - files = files_parameters.split(';') - for layer_file in files: - if not exists(layer_file): - request.appendBody('file not found : %s.\n' % layer_file) + sources = sources_parameters.split(';') + for layer_source in sources: + + if not validate_source_uri(layer_source): + request.appendBody( + 'invalid parameter: {0}.\n'.format(layer_source)) return + if is_file_path(layer_source): + if not exists(layer_source): + request.appendBody('file not found : %s.\n' % layer_source) + return + names_parameters = params.get('NAMES', None) if names_parameters: names = names_parameters.split(';') - if len(names) != len(files): + if len(names) != len(sources): request.appendBody( - 'Not same length between NAMES and FILES') + 'Not same length between NAMES and SOURCES') return else: names = [ - splitext(basename(layer_file))[0] for layer_file in files] + splitext(basename(layer_source))[0] + for layer_source in sources] QgsMessageLog.logMessage('Setting up project to %s' % project_path) project = QgsProject.instance() project.setFileName(project_path) + if not overwrite: + project.read() qml_files = [] qgis_layers = [] vector_layers = [] - raster_layer = [] + raster_layers = [] - for layer_name, layer_file in zip(names, files): - if layer_file.endswith(('shp', 'geojson')): - qgis_layer = QgsVectorLayer(layer_file, layer_name, 'ogr') - vector_layers.append(qgis_layer.id()) + for layer_name, layer_source in zip(names, sources): - elif layer_file.endswith(('asc', 'tiff', 'tif')): - qgis_layer = QgsRasterLayer(layer_file, layer_name) - raster_layer.append(qgis_layer.id()) - else: - request.appendBody('Invalid format : %s' % layer_file) + qgis_layer = layer_from_source(layer_source, layer_name) + + if not qgis_layer: + request.appendBody('Invalid format : %s' % layer_source) return if not qgis_layer.isValid(): - request.appendBody('Layer is not valid : %s' % layer_file) + request.appendBody('Layer is not valid : %s' % layer_source) return + if isinstance(qgis_layer, QgsRasterLayer): + raster_layers.append(qgis_layer.id()) + elif isinstance(qgis_layer, QgsVectorLayer): + vector_layers.append(qgis_layer.id()) + else: + request.appendBody('Invalid type : {0} - {1}'.format( + qgis_layer, type(qgis_layer))) + qgis_layers.append(qgis_layer) - qml_file = splitext(layer_file)[0] + '.qml' + qml_file = splitext(layer_source)[0] + '.qml' if exists(qml_file): # Check if there is a QML qml_files.append(qml_file) @@ -145,29 +165,60 @@ def responseComplete(self): style_manager = qgis_layer.styleManager() style_manager.renameStyle('', 'default') - # add basemap to the qgs project for a background on a thumbnail - if params.get('BASEMAP'): - # basemap is comprised of url and name with semicolon as separator - basemap = params.get('BASEMAP').split(';') - if len(basemap) == 2: - qgis_layer = QgsRasterLayer(basemap[0], basemap[1], 'wms') - if not qgis_layer.isValid(): - request.appendBody('%s cannot be found' % basemap[1]) - return - raster_layer.append(qgis_layer.id()) - qgis_layers.append(qgis_layer) - # Add layer to the registry - QgsMapLayerRegistry.instance().addMapLayers(qgis_layers) + if overwrite: + # Insert all new layers + QgsMapLayerRegistry.instance().addMapLayers(qgis_layers) + else: + # Updating rules + # 1. Get existing layer by name + # 2. Compare source, if it is the same, don't update + # 3. If it is a new name, add it + # 4. If same name but different source, then update + + map_registry = QgsMapLayerRegistry.instance() + + for new_layer in qgis_layers: + # Get existing layer by name + current_layer = map_registry.mapLayersByName( + new_layer.name()) + + # If it doesn't exists, add new layer + if not current_layer: + map_registry.addMapLayer(new_layer) + # If it is exists, compare source + else: + current_layer = current_layer[0] + + # Same source, don't update + if current_layer.source() == new_layer.source(): + if isinstance(new_layer, QgsVectorLayer): + vector_layers.remove(new_layer.id()) + vector_layers.append(current_layer.id()) + elif isinstance(new_layer, QgsRasterLayer): + raster_layers.remove(new_layer.id()) + raster_layers.append(current_layer.id()) + + # Different source, update + else: + QgsMessageLog.logMessage('Update {0}'.format( + new_layer.name())) + if isinstance(new_layer, QgsVectorLayer): + project.removeEntry( + 'WFSLayersPrecision', '/{0}'.format( + current_layer.id())) + + map_registry.removeMapLayer(current_layer.id()) + map_registry.addMapLayer(new_layer) if len(vector_layers): - for layer_file in vector_layers: + for layer_source in vector_layers: project.writeEntry( - 'WFSLayersPrecision', '/%s' % layer_file, 8) + 'WFSLayersPrecision', '/%s' % layer_source, 8) project.writeEntry('WFSLayers', '/', vector_layers) - if len(raster_layer): - project.writeEntry('WCSLayers', '/', raster_layer) + if len(raster_layers): + project.writeEntry('WCSLayers', '/', raster_layers) project.write() project.clear() diff --git a/filters/tools.py b/filters/tools.py index db1b62a..eb1b5b0 100644 --- a/filters/tools.py +++ b/filters/tools.py @@ -16,8 +16,12 @@ * * *************************************************************************** """ +import urllib +import urlparse import xml.etree.ElementTree as ET +from qgis.core import QgsVectorLayer, QgsRasterLayer + def generate_legend(layers, project): """Regenerate the XML for the legend. @@ -47,3 +51,112 @@ def generate_legend(layers, project): xml_root = document.getroot() xml_root.append(xml_legend) document.write(project) + + +def is_file_path(uri): + """True if this is a file path. + + :param uri: The uri to check + :type uri: basestring + + :return: Boolean value + :rtype: bool + """ + try: + # light checking, if it starts with '/', + # then it is probably a file path + if uri.startswith('/'): + return True + # Check if it is a proper file:// uri. + # Need to unquote/decode it first + sanitized_uri = urllib.unquote(uri).decode('utf-8') + if sanitized_uri.startswith('file://'): + return True + except Exception: + return False + + +def is_tile_path(uri): + """True if this is a tile path. + + :param uri: The uri to check + :type uri: basestring + + :return: Boolean value + :rtype: bool + """ + try: + # Since this is a uri, unquote/decode it first + sanitized_uri = urllib.unquote(uri).decode('utf-8') + if sanitized_uri.startswith(('http://', 'https://')): + return True + # It might be in the form of query string + query_params = urlparse.parse_qs(sanitized_uri) + query_params_keys = [k.lower() for k in query_params.keys()] + if 'url' in query_params_keys: + return True + except Exception: + return False + + +def validate_source_uri(source_uri): + """Validate a given source uri. + + A source URI for QgsMapLayer is valid if it is a file path: + e.g. "/path/to/layer.shp" + or a WMS/Tile request + e.g. "type=xyz&url=http://tile.osm.org/{z}/{x}/{y}.png?layers=osm" + + :param source_uri: A source URI + :type source_uri: basestring + + :return: Boolean value + :rtype: bool + """ + return is_file_path(source_uri) or is_tile_path(source_uri) + + +def layer_from_source(source_uri, name): + """Return QgsMapLayer from a given source uri. + + :param source_uri: A source URI + :type source_uri: basestring + + :param name: Designated layer name + :type name: basestring + + :return: QgsMapLayer + :rtype: qgis.core.QgsMapLayer + """ + vector_extensions = ('shp', 'geojson') + raster_extensions = ('asc', 'tiff', 'tif', 'geotiff', 'geotif') + + qgis_layer = None + + if is_file_path(source_uri): + + # sanitize source_uri + sanitized_uri = urllib.unquote(source_uri).decode('utf-8') + sanitized_uri.replace('file://', '') + + if source_uri.endswith(vector_extensions): + qgis_layer = QgsVectorLayer(sanitized_uri, name, 'ogr') + + elif source_uri.endswith(raster_extensions): + qgis_layer = QgsRasterLayer(sanitized_uri, name) + + elif is_tile_path(source_uri): + + # sanitize source_uri + sanitized_uri = urllib.unquote(source_uri).decode('utf-8') + # Check if it is only a url + if sanitized_uri.startswith(('http://', 'https://')): + # Then it is probably a tile xyz url + sanitized_uri = 'type=xyz&url={0}'.format(sanitized_uri) + # It might be in the form of query string + query_params = urlparse.parse_qs(sanitized_uri) + driver = query_params.get('driver', 'wms') + + qgis_layer = QgsRasterLayer(sanitized_uri, name, driver) + + return qgis_layer