Feat/geometry lod

README.md

@@ -18,7 +18,7 @@ See also our companion project [`pg_tileserv`](https://github.com/CrunchyData/pg
 * Implements the [*OGC API - Features*](https://ogcapi.ogc.org/features/) standard.
   * Standard query parameters: `limit`, `bbox`, `bbox-crs`, property filtering, `sortby`, `crs`
   * Query parameters `filter` and `filter-crs` allow [CQL filtering](https://portal.ogc.org/files/96288), with spatial support
-  * Extended query parameters: `offset`, `properties`, `transform`, `precision`, `groupby`
+  * Extended query parameters: `offset`, `properties`, `transform`, `precision`, `groupby`, `max-allowable-offset`
 * Data responses are formatted in JSON and [GeoJSON](https://www.rfc-editor.org/rfc/rfc7946.txt)
 * Provides a simple HTML user interface, with web maps to view spatial data
 * Uses the power of PostgreSQL to reduce the amount of code
@@ -28,6 +28,7 @@ See also our companion project [`pg_tileserv`](https://github.com/CrunchyData/pg
 * Uses PostGIS to provide geospatial functionality:
   * Spatial filtering
   * Transforming geometry data into the output coordinate system
+  * Providing geometry simplification capability
   * Marshalling feature data into GeoJSON
 * Full-featured HTTP support
   * CORS support with configurable Allowed Origins

hugo/content/examples/ex_query_data.md

@@ -82,3 +82,15 @@ http://localhost:9001/collections/ne.countries/items/55.html
 ?properties=gid,name,continent
 ```
 ![Map view of query for feature by ID](/ex-query-data-countries-feature.png)
+
+## Query Features with geometry simplification
+
+When simplifying the returned geometries is needed, you can query one or several features using the `max-allowable-offset` parameter.
+
+This parameter value is interpreted in the unit which corresponds to the output coordinate system.
+The simplification operation is done using the Douglas-Peucker algorithm.
+
+```
+http://localhost:9001/collections/ne.countries/items/55
+?max-allowable-offset=0.1
+```

hugo/content/roadmap/capabilities.md

@@ -64,6 +64,7 @@ It includes [*OGC API - Features*](http://docs.opengeospatial.org/is/17-069r3/17
 - [ ] convert transform function names to `ST_` equivalents
 - [x] `groupBy=colname` to group by column (used with a `transform` spatial aggregate function)
 - [ ] `f` parameter for formats?  (e.g. `f=json`, `f=html`)
+- [x] `max-allowable-offset=tolerance` geometry simplification (Douglas-Peucker algorithm)
 
 ### Query parameters - Functions
 

internal/api/api.go

@@ -93,20 +93,21 @@ const (
 // ================== ParamReserved ==================
 
 const (
-	ParamCrs        = "crs"
-	ParamLimit      = "limit"
-	ParamOffset     = "offset"
-	ParamBbox       = "bbox"
-	ParamBboxCrs    = "bbox-crs"
-	ParamFilter     = "filter"
-	ParamFilterCrs  = "filter-crs"
-	ParamGroupBy    = "groupby"
-	ParamOrderBy    = "orderby"
-	ParamPrecision  = "precision"
-	ParamProperties = "properties"
-	ParamSortBy     = "sortby"
-	ParamTransform  = "transform"
-	ParamType       = "type"
+	ParamCrs                = "crs"
+	ParamLimit              = "limit"
+	ParamOffset             = "offset"
+	ParamBbox               = "bbox"
+	ParamBboxCrs            = "bbox-crs"
+	ParamFilter             = "filter"
+	ParamFilterCrs          = "filter-crs"
+	ParamGroupBy            = "groupby"
+	ParamOrderBy            = "orderby"
+	ParamPrecision          = "precision"
+	ParamProperties         = "properties"
+	ParamSortBy             = "sortby"
+	ParamTransform          = "transform"
+	ParamType               = "type"
+	ParamMaxAllowableOffset = "max-allowable-offset"
 )
 
 // known query parameter name

internal/api/datatype.go

@@ -24,6 +24,7 @@ import (
 
 type PGType string
 type JSONType string
+type GeometryType string
 
 // Constants
 const (
@@ -38,7 +39,6 @@ const (
 	JSONTypeNumberArray  JSONType = "number[]"
 )
 
-// Constants
 const (
 	PGTypeBool         PGType = "bool"
 	PGTypeBoolArray    PGType = "_bool"
@@ -65,6 +65,15 @@ const (
 	PGTypeVarCharArray PGType = "_varchar"
 )
 
+const (
+	GeometryTypePoint           = "Point"
+	GeometryTypeMultiPoint      = "MultiPoint"
+	GeometryTypeLineString      = "LineString"
+	GeometryTypeMultiLineString = "MultiLineString"
+	GeometryTypePolygon         = "Polygon"
+	GeometryTypeMultiPolygon    = "MultiPolygon"
+)
+
 // returns JSONType matching PGType
 func (dbType PGType) ToJSONType() JSONType {
 	//fmt.Printf("ToJSONType: %v\n", pgType)

internal/api/openapi.go

@@ -500,6 +500,23 @@ func GetOpenAPIContent(urlBase string) *openapi3.T {
 			AllowEmptyValue: false,
 		},
 	}
+	paramMaxAllowableOffset := openapi3.ParameterRef{
+		Value: &openapi3.Parameter{
+			Name:        "max-allowable-offset",
+			Description: "Tolerance to apply for geometry simplification on returned feature(s).",
+			In:          "query",
+			Required:    false,
+			Schema: &openapi3.SchemaRef{
+				Value: &openapi3.Schema{
+					Type:    "number",
+					Min:     openapi3.Float64Ptr(0),
+					Max:     openapi3.Float64Ptr(float64(conf.Configuration.Paging.LimitMax)),
+					Default: conf.Configuration.Paging.LimitDefault,
+				},
+			},
+			AllowEmptyValue: false,
+		},
+	}
 	paramOffset := openapi3.ParameterRef{
 		Value: &openapi3.Parameter{
 			Name:        "offset",
@@ -710,6 +727,7 @@ func GetOpenAPIContent(urlBase string) *openapi3.T {
 						&paramCrs,
 						&paramLimit,
 						&paramOffset,
+						&paramMaxAllowableOffset,
 						/* TODO
 						&openapi3.ParameterRef{
 							Value: &openapi3.Parameter{
@@ -808,6 +826,7 @@ func GetOpenAPIContent(urlBase string) *openapi3.T {
 						&paramProperties,
 						&paramTransform,
 						&paramCrs,
+						&paramMaxAllowableOffset,
 					},
 					Responses: openapi3.Responses{
 						"200": &openapi3.ResponseRef{

internal/data/catalog.go

@@ -95,9 +95,10 @@ type QueryParam struct {
 	FilterSql string
 	Filter    []*PropertyFilter
 	// Columns is the list of columns to return
-	Columns       []string
-	GroupBy       []string
-	SortBy        []api.Sorting
-	Precision     int
-	TransformFuns []api.TransformFunction
+	Columns            []string
+	GroupBy            []string
+	SortBy             []api.Sorting
+	Precision          int
+	TransformFuns      []api.TransformFunction
+	MaxAllowableOffset float64
 }

internal/data/db_sql.go

@@ -308,11 +308,19 @@ const sqlFmtGeomCol = `ST_AsGeoJSON( %v %v ) AS _geojson`
 func sqlGeomCol(geomCol string, sourceSRID int, param *QueryParam) string {
 	geomColSafe := strconv.Quote(geomCol)
 	geomExpr := applyTransform(param.TransformFuns, geomColSafe)
-	geomOutExpr := transformToOutCrs(geomExpr, sourceSRID, param.Crs)
+	simplifiedGeom := simplifyWithTolerance(geomExpr, param.MaxAllowableOffset)
+	geomOutExpr := transformToOutCrs(simplifiedGeom, sourceSRID, param.Crs)
 	sql := fmt.Sprintf(sqlFmtGeomCol, geomOutExpr, sqlPrecisionArg(param.Precision))
 	return sql
 }
 
+func simplifyWithTolerance(geomOutExpr string, tolerance float64) string {
+	if tolerance == 0.0 {
+		return geomOutExpr
+	}
+	return fmt.Sprintf("ST_Simplify(%v, %v)", geomOutExpr, tolerance)
+}
+
 func transformToOutCrs(geomExpr string, sourceSRID, outSRID int) string {
 	if sourceSRID == outSRID {
 		return geomExpr

internal/data/feature_mock.go

@@ -55,6 +55,32 @@ func makeFeatureMockPoint(tableName string, id int, x float64, y float64) *featu
 	return &feat
 }
 
+func makeFeatureMockPolygon(tableName string, id int, coords orb.Ring) *featureMock {
+
+	geom := geojson.NewGeometry(orb.Polygon{coords})
+
+	sum := fnv.New32a()
+	encodedContent, _ := json.Marshal(geom)
+	sum.Write(encodedContent)
+	weakEtag := fmt.Sprint(sum.Sum32())
+
+	httpDateString := api.GetCurrentHttpDate() // Last modified value
+
+	idstr := strconv.Itoa(id)
+
+	feat := featureMock{
+		GeojsonFeatureData: *api.MakeGeojsonFeature(
+			tableName,
+			idstr,
+			*geom,
+			map[string]interface{}{"prop_a": "propA", "prop_b": id, "prop_c": "propC", "prop_d": id % 10},
+			weakEtag,
+			httpDateString,
+		),
+	}
+	return &feat
+}
+
 func (fm *featureMock) toJSON(propNames []string) string {
 	props := fm.extractProperties(propNames)
 	return api.MakeGeojsonFeatureJSON("", fm.ID, *fm.Geom, props, fm.WeakEtag.Etag, fm.WeakEtag.LastModified)
@@ -130,11 +156,21 @@ func doLimit(features []*featureMock, limit int, offset int) []*featureMock {
 	return features[start:end]
 }
 
+// Returns a JSON representation of a Point typed feature
 func MakeFeatureMockPointAsJSON(tableName string, id int, x float64, y float64, columns []string) string {
 	feat := makeFeatureMockPoint(tableName, id, x, y)
 	return feat.toJSON(columns)
 }
 
+// Returns a JSON representation of a Polygon typed feature
+func MakeFeatureMockPolygonAsJSON(tableName string, id int, coords orb.Ring, columns []string) string {
+	feat := makeFeatureMockPolygon(tableName, id, coords)
+	return feat.toJSON(columns)
+}
+
+// Generates and returns a slice of Point typed features
+// -> which coordinates are generated inside the provided extent
+// -> which quantity depends on the nx and ny values provided as arguments (nx*ny)
 func MakeFeaturesMockPoint(tableName string, extent api.Extent, nx int, ny int) []*featureMock {
 	basex := extent.Minx
 	basey := extent.Miny
@@ -156,3 +192,21 @@ func MakeFeaturesMockPoint(tableName string, extent api.Extent, nx int, ny int)
 	}
 	return features
 }
+
+// Returns a slice of Polygon typed featureMocks
+func MakeFeaturesMockPolygon(tableName string) []*featureMock {
+
+	polygons := make([]orb.Ring, 0)
+	polygons = append(polygons, (orb.Ring{{-0.024590485281003, 49.2918461864342}, {-0.02824214022877, 49.2902093052715}, {-0.032731597583892, 49.2940548086905}, {-0.037105514267367, 49.2982628947696}, {-0.035096222035489, 49.2991273714187}, {-0.038500457450357, 49.3032655348948}, {-0.034417965728768, 49.3047607558599}, {-0.034611922456059, 49.304982637632}, {-0.028287271276391, 49.3073904622151}, {-0.022094153540685, 49.3097046833446}, {-0.022020905508067, 49.3096240670749}, {-0.019932810088915, 49.3103884833526}, {-0.013617304476105, 49.3129751788625}, {-0.010317714854534, 49.3091925467367}, {-0.006352474569531, 49.3110873002743}, {-0.001853050940172, 49.3070612288807}, {0.002381370562776, 49.3028484930665}, {-0.000840217324783, 49.3013882187799}, {-0.00068928216257, 49.3012429006019}, {-0.003864625123604, 49.3000173218511}, {-0.003918013833785, 49.2999931219338}, {-0.010095065847337, 49.2974103246769}, {-0.010150643294152, 49.2974622610823}, {-0.013587537856462, 49.2959737733625}, {-0.01384030494609, 49.2962233671643}, {-0.017222409797967, 49.294623513139}, {-0.017308576106142, 49.2947057553981}, {-0.020709238582055, 49.2930969232562}, {-0.021034503634088, 49.2933909821512}, {-0.024481057600533, 49.2917430023163}, {-0.024590485281003, 49.2918461864342}}))
+	polygons = append(polygons, (orb.Ring{{0.012754827133148, 49.3067879156925}, {0.008855271114669, 49.3050781328888}, {0.004494239224312, 49.3091080209745}, {-0.000152707581678, 49.3133105602284}, {0.005720060734669, 49.3160862415579}, {0.005012790172897, 49.3167672210029}, {0.000766997696737, 49.3211596408574}, {0.007624129875227, 49.3239385018443}, {0.008367761372595, 49.3242455690107}, {0.008290411160612, 49.3243148348313}, {0.014857908580632, 49.327355944666}, {0.021563621634322, 49.330400077634}, {0.021666104647453, 49.3302974189836}, {0.024971410363691, 49.3317809883673}, {0.02492195583839, 49.3318321743075}, {0.029104098429698, 49.3336152412767}, {0.028646253682028, 49.3340827604102}, {0.035511767129074, 49.3367701742839}, {0.04198105053544, 49.3391776115466}, {0.046199095420336, 49.3352329627991}, {0.047069675744848, 49.3344290720305}, {0.048144047016136, 49.334920703514}, {0.048423560249958, 49.3346968337392}, {0.051915791431139, 49.3363621210079}, {0.056947292176151, 49.3326168697662}, {0.061993411180365, 49.3286019089077}, {0.055850651601917, 49.3253039337471}, {0.049713813923233, 49.3219158062857}, {0.049393633537099, 49.3221688494924}, {0.047471649153311, 49.3213066024438}, {0.04755106595679, 49.3212332612062}, {0.040845011450398, 49.3181905415208}, {0.040150920245632, 49.31787904142}, {0.039962885130089, 49.317782152465}, {0.04034174516319, 49.3173686114171}, {0.033626289449895, 49.3145051363955}, {0.032740557919845, 49.3141516109565}, {0.031347338613429, 49.313459605015}, {0.031235682243362, 49.3135509641281}, {0.029314267528688, 49.3127840624681}, {0.024083333873085, 49.3105820713374}, {0.02383988821816, 49.3108046457384}, {0.022989404102509, 49.3104651415232}, {0.016397609318679, 49.3078735624598}, {0.016236244414416, 49.3080276777805}, {0.013035870818624, 49.3065310213615}, {0.012754827133148, 49.3067879156925}}))
+	polygons = append(polygons, (orb.Ring{{0.019797816099279, 49.325229088603}, {0.013235498621243, 49.3220984135413}, {0.006679188663454, 49.3188775447307}, {0.001751478001915, 49.3231631269776}, {0.00030826510927, 49.3244180023312}, {0.000034521402383, 49.3242899085418}, {-0.004894257776504, 49.3285751953461}, {-0.009823855515987, 49.332860261738}, {-0.003845879462176, 49.3357402000546}, {-0.004376904724334, 49.336234279179}, {0.00019267127677, 49.3382699850882}, {0.00003896662097, 49.3384130063648}, {0.006882712504834, 49.3414613328914}, {0.013584586312611, 49.3445956881043}, {0.013835900545075, 49.3443662391223}, {0.018429968444473, 49.3465144456831}, {0.019007858697842, 49.3459970497808}, {0.022212104736706, 49.3477771230593}, {0.028477356337026, 49.3513495867644}, {0.033807665316216, 49.347252820989}, {0.038724697445692, 49.3431456923271}, {0.034812389120157, 49.3408267818312}, {0.036339781995501, 49.3391292768443}, {0.040721479048813, 49.3347390581568}, {0.036808655724018, 49.3329836158413}, {0.037123735821512, 49.3326718720873}, {0.030269026676719, 49.3298048842398}, {0.023282829964216, 49.3268442840858}, {0.023162342964376, 49.3269672904862}, {0.021527329925941, 49.3262612666818}, {0.019602511201379, 49.3254039935278}, {0.019797816099279, 49.325229088603}}))
+
+	id := 100 // arbitrary value used to populate the feature properties
+
+	features := make([]*featureMock, 0)
+	for _, coords := range polygons {
+		feature := makeFeatureMockPolygon(tableName, id, coords)
+		features = append(features, feature)
+	}
+	return features
+}

internal/service/db_test/handler_db_lod_test.go

@@ -0,0 +1,106 @@
+package db_test
+
+/*
+ Copyright 2023 Crunchy Data Solutions, Inc.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ Date     : February 2023
+ Authors  : Nicolas Revelant (nicolas dot revelant at ign dot fr)
+*/
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"testing"
+
+	"github.com/CrunchyData/pg_featureserv/internal/api"
+	util "github.com/CrunchyData/pg_featureserv/internal/utiltest"
+	"github.com/paulmach/orb"
+	"github.com/paulmach/orb/geojson"
+)
+
+// Simple unit test case ensuring that simplification is working on a single feature
+func (t *DbTests) TestGeometrySimplificationSingleFeature() {
+	t.Test.Run("TestGeometrySimplificationSingleFeature", func(t *testing.T) {
+
+		rr := hTest.DoRequest(t, "/collections/mock_poly/items/1?max-allowable-offset=0.01")
+
+		var feat api.GeojsonFeatureData
+		errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &feat)
+		util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh))
+		util.Equals(t, 4, len(feat.Geom.Geometry().(orb.Polygon)[0]), "wrong number of simplified coordinates")
+
+	})
+}
+
+// Simple unit test case ensuring that simplification is working on several features
+func (t *DbTests) TestGeometrySimplificationSeveralFeatures() {
+	t.Test.Run("TestGeometrySimplificationSeveralFeatures", func(t *testing.T) {
+
+		rr := hTest.DoRequest(t, "/collections/mock_poly/items?max-allowable-offset=0.01")
+		// Feature collection
+		var v api.FeatureCollection
+		errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v)
+		util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh))
+		util.Assert(t, len(v.Features) > 1, "no features returned after simplification")
+
+		for _, feature := range v.Features {
+			util.Assert(t, len(feature.Geom.Geometry().(orb.Polygon)[0]) > 1, "")
+			util.Assert(t, len(feature.Geom.Geometry().(orb.Polygon)[0]) < 30, "")
+		}
+
+	})
+}
+
+// Test case with negative value as simplification factor
+func (t *DbTests) TestGeometrySimplificationNegativeValue() {
+	t.Test.Run("TestGeometrySimplificationNegativeValue", func(t *testing.T) {
+		path := "/collections/mock_poly/items/1?max-allowable-offset=-0.01"
+		// If lower than minVal, then minValue (0) is considered
+		hTest.DoRequestMethodStatus(t, "GET", path, nil, nil, http.StatusOK)
+	})
+}
+
+// Test case with wrong float separator for the simplification factor
+func (t *DbTests) TestGeometrySimplificationWrongFloatSeparatorValue() {
+	t.Test.Run("TestGeometrySimplificationWrongFloatSeparatorValue", func(t *testing.T) {
+		path := "/collections/mock_poly/items?max-allowable-offset=0,01"
+		hTest.DoRequestMethodStatus(t, "GET", path, nil, nil, http.StatusBadRequest)
+	})
+}
+
+// Test case with various values as simplification factor
+func (t *DbTests) TestGeometrySimplificationVariousSimplificationValues() {
+	t.Test.Run("TestGeometrySimplificationVariousSimplificationValues", func(t *testing.T) {
+		path := "/collections/mock_poly/items/1?max-allowable-offset=0.01"
+		rr := hTest.DoRequestMethodStatus(t, "GET", path, nil, nil, http.StatusOK)
+		// Feature collection
+		var feat api.GeojsonFeatureData
+		errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &feat)
+		util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh))
+		util.Equals(t, 4, len(feat.Geom.Geometry().(orb.Polygon)[0]), "wrong number of simplified coordinates")
+
+		path = "/collections/mock_poly/items/1?max-allowable-offset=0.001"
+		rr = hTest.DoRequestMethodStatus(t, "GET", path, nil, nil, http.StatusOK)
+		errUnMarsh = json.Unmarshal(hTest.ReadBody(rr), &feat)
+		util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh))
+		util.Equals(t, 10, len(feat.Geom.Geometry().(orb.Polygon)[0]), "wrong number of simplified coordinates")
+
+		path = "/collections/mock_poly/items/1?max-allowable-offset=1"
+		rr = hTest.DoRequestMethodStatus(t, "GET", path, nil, nil, http.StatusOK)
+		errUnMarsh = json.Unmarshal(hTest.ReadBody(rr), &feat)
+		util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh))
+		util.Equals(t, (*geojson.Geometry)(nil), feat.Geom, "simplified geometry still present")
+
+	})
+}

internal/service/db_test/runner_db_test.go

@@ -163,6 +163,16 @@ func TestRunnerHandlerDb(t *testing.T) {
 		test.TestReplaceFeatureIfNoneMatchStarValueWithExistingRepresentationInCacheDb()
 		afterEachRun()
 	})
+	t.Run("LOD", func(t *testing.T) {
+		beforeEachRun()
+		test := DbTests{Test: t}
+		test.TestGeometrySimplificationSingleFeature()
+		test.TestGeometrySimplificationSeveralFeatures()
+		test.TestGeometrySimplificationNegativeValue()
+		test.TestGeometrySimplificationWrongFloatSeparatorValue()
+		test.TestGeometrySimplificationVariousSimplificationValues()
+		afterEachRun()
+	})
 
 	t.Run("SPECIAL_SCHEMA_TABLE_COLUMN", func(t *testing.T) {
 		beforeEachRun()

internal/service/handler.go

@@ -465,7 +465,6 @@ func handleCollectionItems(w http.ResponseWriter, r *http.Request) *appError {
 	}
 	param, errQuery := createQueryParams(&reqParam, tbl.Columns, tbl.Srid)
 	param.Filter = parseFilter(reqParam.Values, tbl.DbTypes)
-
 	if errQuery == nil {
 		ctx := r.Context()
 		switch format {