From 9d0b52e392de17e799c810084fcad218200a4922 Mon Sep 17 00:00:00 2001 From: Maxim Rylov Date: Fri, 2 Dec 2022 13:19:05 +0100 Subject: [PATCH] HANA: Implement NewMVTTileProvider --- mvtprovider/hana/README.md | 132 ++++++++++++++++++++++++++++++++++++ mvtprovider/hana/doc.go | 5 ++ provider/hana/hana.go | 133 ++++++++++++++++++++++++++++++++++++- provider/hana/hana_test.go | 119 ++++++++++++++++++++++++++++++++- provider/hana/register.go | 7 +- provider/hana/util.go | 30 +++++++++ 6 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 mvtprovider/hana/README.md create mode 100644 mvtprovider/hana/doc.go diff --git a/mvtprovider/hana/README.md b/mvtprovider/hana/README.md new file mode 100644 index 000000000..dc60c66a1 --- /dev/null +++ b/mvtprovider/hana/README.md @@ -0,0 +1,132 @@ +# HANA MVT Provider + +The HANA MVT provider manages querying for tile requests against a HANA database with [Vector Tiles support](https://help.sap.com/docs/HANA_CLOUD_DATABASE/bc9e455fe75541b8a248b4c09b086cf5/8cd683c4bb664fd8a71fc3f19ffa7e42.html) to handle the MVT encoding at the database. + +The connection between tegola and HANA is configured in a `tegola.toml` file. An example minimum connection config: + + +```toml +[[providers]] +name = "test_hana" # provider name is referenced from map layers (required) +type = "mvt_hana" # the type of data provider must be "mvt_hana" for this data provider (required) +uri = "hdb://myuser:mypassword@something.hanacloud.ondemand.com:443?" # HANA connection string (required) +``` + +### Connection Properties + +- `uri` (string): [Required] HANA connection string +- `name` (string): [Required] provider name is referenced from map layers +- `type` (string): [Required] the type of data provider. must be "hana" to use this data provider +- `srid` (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) + +#### Connection string properties + +**Example** + +``` +# {protocol}://{user}:{password}@{host}:{port}/{database}?{options}= +hdb://myuser:mypassword@something.hanacloud.ondemand.com:443?TLSInsecureSkipVerify&timeout=3600&max_connections=30 +``` + +**Options** + +- `timeout`: [Optional] Driver side connection timeout in seconds. +- `TLSRootCAFile` [Optional] Path,- filename to root certificate(s). +- `TLSServerName` [Optional] ServerName to verify the hostname. By setting TLSServerName=host, the provider will set TLSServerName same as 'host' value in `uri`. +- `TLSInsecureSkipVerify` [Optional] Controls whether a client verifies the server's certificate chain and host name. +- `max_connections`: [Optional] The max connections to maintain in the connection pool. Defaults to 100. 0 means no max. +- `max_connection_idle_time`: [Optional] The maximum time an idle connection is kept alive. Defaults to "30m". +- `max_connection_life_time` [Optional] The maximum time a connection lives before it is terminated and recreated. Defaults to "1h". + +## Provider Layers + +In addition to the connection configuration above, Provider Layers need to be configured. A Provider Layer tells tegola how to query HANA for a certain layer. When using the HANA MVT Provider the `ST_AsMVTGeom()` MUST be used. An example minimum config using the `sql` config option: + +```toml +[[providers.layers]] +name = "landuse" +# MVT data provider can use both table names and SQL statements. Internally, the provider wraps an SQL query by using +# ST_AsMVT and ST_AsMVTGeom functions. +tablename = "gis.zoning_base_3857" +``` + +### Provider Layers Properties + +- `name` (string): [Required] the name of the layer. This is used to reference this layer from map layers. +- `geometry_fieldname` (string): [Optional] the name of the filed which contains the geometry for the feature. Defaults to `geom`. +- `id_fieldname` (string): [Optional] the name of the feature id field. defaults to `gid`. +- `geometry_type` (string): [Optional] the layer geometry type. If not set, the table will be inspected at startup to try and infer the gemetry type. Valid values are: `Point`, `LineString`, `Polygon`, `MultiPoint`, `MultiLineString`, `MultiPolygon`, `GeometryCollection`. +- `srid` (int): [Optional] the SRID of the layer. Supports `3857` (WebMercator) only. +- `sql` (string): [Required] custom SQL to use use. Supports the following tokens: + - `!BBOX!` - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. `!bbox!` and`!BOX!` are supported as well for compatibilitiy with queries from Mapnik and MapServer styles. + - `!X!` - [Optional] will replaced with the "X" value of the requested tile. + - `!Y!` - [Optional] will replaced with the "Y" value of the requested tile. + - `!Z!` - [Optional] will replaced with the "Z" value of the requested tile. + - `!ZOOM!` - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. + - `!SCALE_DENOMINATOR!` - [Optional] scale denominator, assuming 90.7 DPI (i.e. 0.28mm pixel size) + - `!PIXEL_WIDTH!` - [Optional] the pixel width in meters, assuming 256x256 tiles + - `!PIXEL_HEIGHT!` - [Optional] the pixel height in meters, assuming 256x256 tiles + - `!ID_FIELD!` - [Optional] the id field name + - `!GEOM_FIELD!` - [Optional] the geom field name + - `!GEOM_TYPE!` - [Optional] the geom type if defined otherwise "" +- `buffer` (int): [Optional] the buffer distance by which the clipped geometry may exceed the tile's area. Defaults to 256. +- `clip_geometry` (bool): [Optional] the flag to control whether the geometry is clipped to the tile bounds or not. Defaults to `TRUE`. + +## Example mvt_hana and map config + +```toml +[[providers]] +name = "test_hana" +type = "mvt_hana" +uri = "hdb://myuser:mypassword@something.hanacloud.ondemand.com:443?" # HANA connection string (required) +srid = 3857 # The only supported srid is 3857 (optional) + + [[providers.layers]] + name = "landuse" + sql = "SELECT geom, gid FROM gis.landuse WHERE !BBOX!" + +[[maps]] +name = "cities" +center = [-90.2,38.6,3.0] # where to center of the map (lon, lat, zoom) + + [[maps.layers]] + name = "landuse" + provider_layer = "test_hana.landuse" + min_zoom = 0 + max_zoom = 14 +``` + +## Example mvt_hana and map config for SRID 4326 + +When using a 4326 projection with ST_AsMVT the SQL statement needs to be modified. `ST_AsMVTGeom` is expecting data in 3857 projection so the geometries and the `!BBOX!` token need to be transformed prior to `ST_AsMVTGeom` processing them. For example: + +```toml +[[providers]] +name = "test_hana" +type = "mvt_hana" +uri = "hdb://myuser:mypassword@something.hanacloud.ondemand.com:443?" # HANA connection string (required) +srid = 3857 # The only supported srid is 3857 (optional) + + [[providers.layers]] + name = "landuse" + sql = "SELECT * FROM (SELECT id, name, geom.ST_Transform(3857) AS geom FROM ne_50m_rivers_lake_centerlines) AS sub WHERE !BBOX!" + +[[maps]] +name = "cities" +center = [-90.2,38.6,3.0] # where to center of the map (lon, lat, zoom) + + [[maps.layers]] + name = "landuse" + provider_layer = "test_hana.landuse" + min_zoom = 0 + max_zoom = 14 +``` + +## Testing + +Testing is designed to work against a live SAP HANA database. To see how to set up a database check this [github actions script](https://github.com/go-spatial/tegola/blob/master/.github/worksflows/on_pr_push.yml). To run the HANA tests, the following environment variables need to be set: + +```bash +$ export RUN_HANA_TESTS=yes +$ export HANA_CONNECTION_STRING="hdb://myuser:mypassword@something.hanacloud.ondemand.com:443?TLSInsecureSkipVerify" +``` \ No newline at end of file diff --git a/mvtprovider/hana/doc.go b/mvtprovider/hana/doc.go new file mode 100644 index 000000000..7b93ac102 --- /dev/null +++ b/mvtprovider/hana/doc.go @@ -0,0 +1,5 @@ +// Package hana is a placeholder for the SAP HANA database. +// The hana provider is provided by the provider.Hana provider +// please consult that for information on how to use it. +// please note that the provider type will be mvt_hana +package hana diff --git a/provider/hana/hana.go b/provider/hana/hana.go index 6306d80e6..740760876 100644 --- a/provider/hana/hana.go +++ b/provider/hana/hana.go @@ -1,6 +1,7 @@ package hana import ( + "bytes" "context" "database/sql" "fmt" @@ -140,7 +141,8 @@ type Provider struct { collectorsRegistered bool // Collectors for Query times - queryHistogramSeconds *prometheus.HistogramVec + mvtProviderQueryHistogramSeconds *prometheus.HistogramVec + queryHistogramSeconds *prometheus.HistogramVec } func (p *Provider) Collectors(prefix string, cfgFn func(configKey string) map[string]interface{}) ([]observability.Collector, error) { @@ -154,6 +156,15 @@ func (p *Provider) Collectors(prefix string, cfgFn func(configKey string) map[st return nil, err } + p.mvtProviderQueryHistogramSeconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: prefix + "_mvt_provider_sql_query_seconds", + Help: "A histogram of query time for sql for mvt providers", + Buckets: buckets, + }, + []string{"map_name", "z"}, + ) + p.queryHistogramSeconds = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: prefix + "_provider_sql_query_seconds", @@ -164,7 +175,7 @@ func (p *Provider) Collectors(prefix string, cfgFn func(configKey string) map[st ) p.collectorsRegistered = true - return append(collectors, p.queryHistogramSeconds), nil + return append(collectors, p.mvtProviderQueryHistogramSeconds, p.queryHistogramSeconds), nil } const ( @@ -363,6 +374,9 @@ func CreateProvider(config dict.Dicter, maps []provider.Map, providerType string } majorVersion, _ := strconv.Atoi(strings.Split(dbVersion, ".")[0]) + if providerType == MVTProviderType && majorVersion < 4 { + return nil, fmt.Errorf("MVT provider is only available in HANA Cloud") + } srid := -1 if srid, err = config.Int(ConfigKeySRID, &srid); err != nil { @@ -538,6 +552,23 @@ func CreateProvider(config dict.Dicter, maps []provider.Map, providerType string } } + if providerType == MVTProviderType { + var buffer uint = 256 + if buffer, err = layer.Uint(ConfigKeyBuffer, &buffer); err != nil { + return nil, err + } + + var clipGeom bool = true + if clipGeom, err = layer.Bool(ConfigKeyClipGeometry, &clipGeom); err != nil { + return nil, err + } + + l.sql, err = genMVTSQL(&l, getFieldNames(l.fields), buffer, clipGeom) + if err != nil { + return nil, err + } + } + if debugLayerSQL { log.Debugf("SQL for Layer(%v):\n%v\n", lName, l.sql) } @@ -800,6 +831,104 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return rows.Err() } +func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params provider.Params, layers []provider.Layer) ([]byte, error) { + var mapName string + + { + mapNameVal := ctx.Value(observability.ObserveVarMapName) + if mapNameVal != nil { + // if it's not convertible to a string, we will ignore it. + mapName, _ = mapNameVal.(string) + } + } + + args := make([]interface{}, 0) + var mvtBytes bytes.Buffer + var totalSeconds float64 + totalSeconds = 0.0 + + for i := range layers { + layer := layers[i] + if debug { + log.Debugf("looking for layer: %v", layer) + } + l, ok := p.Layer(layer.Name) + + if !ok { + // Should we error here, or have a flag so that we don't + // spam the user? + log.Warnf("provider layer not found %v", layer.Name) + } + if debugLayerSQL { + log.Debugf("SQL for Layer(%v):\n%v\n", l.Name(), l.sql) + } + + sqlQuery, err := replaceTokens(p.dbVersion, l.sql, l.IDFieldName(), l.GeomFieldName(), l.GeomType(), l.SRID(), tile, false) + if err := ctxErr(ctx, err); err != nil { + return nil, err + } + + // replace configured query parameters if any + sqlQuery = params.ReplaceParams(sqlQuery, &args) + + now := time.Now() + + extent, _ := getTileExtent(tile, false) + srid := l.SRID() + rows, err := p.pool.QueryContextWithBBox(ctx, sqlQuery, extent, srid, true) + + if err := ctxErr(ctx, err); err != nil { + return []byte{}, err + } + + defer rows.Close() + + if err := ctxErr(ctx, err); err != nil { + return []byte{}, err + } + + lob := &driver.Lob{} + lob.SetWriter(new(bytes.Buffer)) + + if rows.Next() { + if err := ctx.Err(); err != nil { + return []byte{}, err + } + + err = rows.Scan(lob) + if err := ctxErr(ctx, err); err != nil { + return []byte{}, err + } + + mvtBytes.Write(lob.Writer().(*bytes.Buffer).Bytes()) + } else { + return nil, fmt.Errorf("unable to read the result set for layer (%v)", l.Name()) + } + + totalSeconds += time.Since(now).Seconds() + + if debugExecuteSQL { + log.Debugf("%s:%s: %v", EnvSQLDebugName, EnvSQLDebugExecute, sqlQuery) + if err != nil { + log.Errorf("%s:%s: returned error %v", EnvSQLDebugName, EnvSQLDebugExecute, err) + } else { + log.Debugf("%s:%s: returned %v bytes", EnvSQLDebugName, EnvSQLDebugExecute, lob.Writer().(*bytes.Buffer).Len()) + } + } + } + + if p.mvtProviderQueryHistogramSeconds != nil { + z, _, _ := tile.ZXY() + lbls := prometheus.Labels{ + "z": strconv.FormatUint(uint64(z), 10), + "map_name": mapName, + } + p.mvtProviderQueryHistogramSeconds.With(lbls).Observe(totalSeconds) + } + + return mvtBytes.Bytes(), nil +} + // Close will close the Provider's database connectio func (p *Provider) Close() { p.pool.Close() } diff --git a/provider/hana/hana_test.go b/provider/hana/hana_test.go index 22b49714e..b53739653 100644 --- a/provider/hana/hana_test.go +++ b/provider/hana/hana_test.go @@ -242,7 +242,6 @@ func TestTileFeatures(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config() - config[hana.ConfigKeyName] = "provider_name" p, err := hana.NewTileProvider(config, nil) if err != nil { if err == tc.expectedErr { @@ -474,3 +473,121 @@ func TestTileFeatures(t *testing.T) { t.Run(name, fn(tc)) } } + +func TestMVTForLayers(t *testing.T) { + ttools.ShouldSkip(t, TESTENV) + + type tcase struct { + TCConfig + layerNames []string + mvtTile []byte + err string + tile provider.Tile + } + + fn := func(tc tcase) func(t *testing.T) { + return func(t *testing.T) { + config := tc.Config() + prvd, err := hana.NewMVTTileProvider(config, nil) + // for now we will just check the length of the bytes. + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) { + t.Logf("error %#v", err) + t.Errorf("expected error with %v in NewMVTTileProvider, got: %v", tc.err, err) + } + return + } + if err != nil { + t.Errorf("NewMVTTileProvider unexpected error: %v", err) + return + } + layers := make([]provider.Layer, len(tc.layerNames)) + + for i := range tc.layerNames { + layers[i] = provider.Layer{ + Name: tc.layerNames[i], + MVTName: tc.layerNames[i], + } + } + mvtTile, err := prvd.MVTForLayers(context.Background(), tc.tile, nil, layers) + if err != nil { + t.Errorf("NewProvider unexpected error: %v", err) + return + } + if len(tc.mvtTile) != len(mvtTile) { + t.Errorf("tile byte length, expected %v got %v", len(tc.mvtTile), len(mvtTile)) + } + } + } + tests := map[string]tcase{ + "SQL with fields and id": { + TCConfig: TCConfig{ + LayerConfig: []map[string]interface{}{ + { + hana.ConfigKeyFeatureIDField: "id", + hana.ConfigKeyGeomType: "multilinestring", + hana.ConfigKeyGeomField: "geom", + hana.ConfigKeyLayerName: "rivers", + hana.ConfigKeySQL: `SELECT * FROM (SELECT "id", "featurecla", "geom".ST_Transform(3857) AS "geom" FROM "TEGOLACI"."ne_50m_rivers_lake_centerlines") AS sub WHERE !BBOX!`, + hana.ConfigKeySRID: 3857, + }, + }, + }, + layerNames: []string{"rivers"}, + mvtTile: make([]byte, 7619), + tile: provider.NewTile(2, 1, 1, 16, 4326), + }, + "SQL with fields and without id": { + TCConfig: TCConfig{ + LayerConfig: []map[string]interface{}{ + { + hana.ConfigKeyGeomType: "multilinestring", + hana.ConfigKeyGeomField: "geom", + hana.ConfigKeyLayerName: "rivers", + hana.ConfigKeySQL: `SELECT * FROM (SELECT "id", "featurecla", "geom".ST_Transform(3857) AS "geom" FROM "TEGOLACI"."ne_50m_rivers_lake_centerlines") AS sub WHERE !BBOX!`, + hana.ConfigKeySRID: 3857, + }, + }, + }, + layerNames: []string{"rivers"}, + mvtTile: make([]byte, 7436), + tile: provider.NewTile(2, 1, 1, 16, 4326), + }, + "SQL with id only": { + TCConfig: TCConfig{ + LayerConfig: []map[string]interface{}{ + { + hana.ConfigKeyFeatureIDField: "id", + hana.ConfigKeyGeomType: "multilinestring", + hana.ConfigKeyGeomField: "geom", + hana.ConfigKeyLayerName: "rivers", + hana.ConfigKeySQL: `SELECT * FROM (SELECT "id", "geom".ST_Transform(3857) AS "geom" FROM "TEGOLACI"."ne_50m_rivers_lake_centerlines") AS sub WHERE !BBOX!`, + hana.ConfigKeySRID: 3857, + }, + }, + }, + layerNames: []string{"rivers"}, + mvtTile: make([]byte, 7443), + tile: provider.NewTile(2, 1, 1, 16, 4326), + }, + "SQL without any fields": { + TCConfig: TCConfig{ + LayerConfig: []map[string]interface{}{ + { + hana.ConfigKeyGeomType: "multilinestring", + hana.ConfigKeyGeomField: "geom", + hana.ConfigKeyLayerName: "rivers", + hana.ConfigKeySQL: `SELECT * FROM (SELECT "geom".ST_Transform(3857) AS "geom" FROM "TEGOLACI"."ne_50m_rivers_lake_centerlines") AS sub WHERE !BBOX!`, + hana.ConfigKeySRID: 3857, + }, + }, + }, + layerNames: []string{"rivers"}, + mvtTile: make([]byte, 6676), + tile: provider.NewTile(2, 1, 1, 16, 4326), + }, + } + for name, tc := range tests { + t.Run(name, fn(tc)) + } +} diff --git a/provider/hana/register.go b/provider/hana/register.go index e809f9835..e83922dae 100644 --- a/provider/hana/register.go +++ b/provider/hana/register.go @@ -7,10 +7,12 @@ import ( func init() { provider.Register(provider.TypeStd.Prefix()+Name, NewTileProvider, Cleanup) + provider.MVTRegister(provider.TypeMvt.Prefix()+Name, NewMVTTileProvider, Cleanup) } const ( - ProviderType = "hana" + MVTProviderType = "mvt_hana" + ProviderType = "hana" ) // NewTileProvider instantiates and returns a new HANA provider or an error. @@ -38,3 +40,6 @@ const ( func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { return CreateProvider(config, maps, ProviderType) } +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { + return CreateProvider(config, maps, MVTProviderType) +} diff --git a/provider/hana/util.go b/provider/hana/util.go index 7a2983690..712d62cc8 100644 --- a/provider/hana/util.go +++ b/provider/hana/util.go @@ -189,6 +189,36 @@ func genSQL(l *Layer, tblName string, fieldNames []string, buffer bool, provider return fmt.Sprintf(stdSQL, strings.Join(fieldNames, ", "), quoteTableName(tblName)), nil } +func genMVTSQL(l *Layer, fields []string, buffer uint, clipGeometry bool) (sql string, err error) { + var flds []string + for i := range fields { + if l.GeomFieldName() != fields[i] { + flds = append(flds, quoteIdentifier(fields[i])) + } + } + + geomFieldName := quoteIdentifier(l.GeomFieldName()) + + // ref: https://help.sap.com/docs/HANA_CLOUD_DATABASE/bc9e455fe75541b8a248b4c09b086cf5/8cd683c4bb664fd8a71fc3f19ffa7e42.html + // BLOB ST_AsMVT(expression_list Expression List, layer_name NCLOB, extent INT, geom_name NCLOB, feature_id_name NCLOB) + + var clip string = "TRUE" + if !clipGeometry { + clip = "FALSE" + } + + if len(flds) == 0 { + sql = fmt.Sprintf(`SELECT ST_AsMVT(%v.ST_AsMVTGeom(bounds => NEW ST_LINESTRING($4, $3), buffer => %v, clipgeom => %v) AS %v, layer_name => '%v', geom_name => '%v') FROM (%v)`, geomFieldName, buffer, clip, geomFieldName, l.Name(), l.GeomFieldName(), l.sql) + } else { + if l.IDFieldName() != "" { + sql = fmt.Sprintf(`SELECT ST_AsMVT(%v, %v.ST_AsMVTGeom(bounds => NEW ST_LINESTRING($4, $3), buffer => %v, clipgeom => %v) AS %v, layer_name => '%v', geom_name => '%v', feature_id_name => '%v') FROM (%v)`, strings.Join(flds, ","), geomFieldName, buffer, clip, geomFieldName, l.Name(), l.GeomFieldName(), l.IDFieldName(), l.sql) + } else { + sql = fmt.Sprintf(`SELECT ST_AsMVT(%v, %v.ST_AsMVTGeom(bounds => NEW ST_LINESTRING($4, $3), buffer => %v, clipgeom => %v) AS %v, layer_name => '%v', geom_name => '%v') FROM (%v)`, strings.Join(flds, ","), geomFieldName, buffer, clip, geomFieldName, l.Name(), l.GeomFieldName(), l.sql) + } + } + return sql, nil +} + const ( PLANAR_SRID_OFFSET = 1000000000 )